diff --git a/Cargo.lock b/Cargo.lock index bb23328cc..933b01efb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -434,12 +434,7 @@ dependencies = [ name = "asap-physical-asap-optimization" version = "0.1.0" dependencies = [ - "asap-accuracy", - "asap-cost", - "asap-logical-asap-optimization", "asap-types", - "rmp-serde", - "serde_json", "thiserror 2.0.18", ] @@ -449,14 +444,9 @@ version = "0.1.0" dependencies = [ "asap-accuracy", "asap-cost", - "asap-frontend-promql", - "asap-logical-asap-optimization", - "asap-physical-asap-optimization", "asap-types", - "asap_sketchlib", "good_lp", "highs", - "rmp-serde", "serde", "serde_json", "thiserror 2.0.18", @@ -476,7 +466,6 @@ dependencies = [ "asap-logical-asap-optimization", "asap-physical-asap-optimization", "asap-plan-selection", - "asap-planner", "asap-types", "futures", "serde", diff --git a/crates/accuracy/src/budget.rs b/crates/accuracy/src/budget.rs index 562529301..65d552250 100644 --- a/crates/accuracy/src/budget.rs +++ b/crates/accuracy/src/budget.rs @@ -14,7 +14,7 @@ //! step's slack. A lone approximate denominator over an exact numerator is //! sized for the whole target, so its quotient's bound `ε/(1 − ε)` misses //! it. Only equal splits are made. -use super::*; +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; /// The target of one of `approximate_inputs` approximate inputs that /// compose into a value with `target`, split equally in `metric`'s diff --git a/crates/accuracy/src/composition.rs b/crates/accuracy/src/composition.rs index 3776f485b..81b647234 100644 --- a/crates/accuracy/src/composition.rs +++ b/crates/accuracy/src/composition.rs @@ -2,7 +2,14 @@ //! Stage 1 composes each candidate's guarantees with them //! (`per_computation_candidates`); a rule that needs a statistic leaves an //! `Unknown` leaf named by one of the constants below, which Stage 3 binds. -use super::*; +use asap_types::accuracy::{ + AccuracyError, AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, + GuaranteeSource, ProbabilityExpr, +}; + +#[cfg(test)] +use crate::requirements::meets; +use crate::PropagationStats; /// Rows an exact aggregation folds (per group): bounded by the series /// count of the sources the query reads. @@ -485,7 +492,10 @@ fn exact_division( /// Whether `guarantee` is a sketch's distinct-count estimate. fn estimates_distinct_count(guarantee: &AccuracyGuarantee) -> bool { - let cardinality = format!("{:?}", asap_types::ir::schema::SketchStatistic::Cardinality); + let cardinality = format!( + "{:?}", + asap_types::summary_type::SketchStatistic::Cardinality + ); guarantee.provenance.iter().any(|source| { matches!(source, GuaranteeSource::SketchEvaluation { query, .. } if *query == cardinality) }) @@ -1135,7 +1145,7 @@ mod tests { s, GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" ))); - assert!(!satisfies(&out, &AccuracyTarget::max_error(1.0))); + assert!(!meets(&out, 1.0, None)); let known = PropagationStats { input_row_count: Some(4), diff --git a/crates/accuracy/src/error_propagation.rs b/crates/accuracy/src/error_propagation.rs new file mode 100644 index 000000000..d3d755f4e --- /dev/null +++ b/crates/accuracy/src/error_propagation.rs @@ -0,0 +1,32 @@ +//! Each operator's local error propagation: the linear bound $g_c$ on a +//! consumer's output error from its inputs' errors (concepts doc, section 4). + +use asap_types::accuracy::AccuracyMetric; +use asap_types::ir::Operator; +use asap_types::stages::MemoGroupId; + +/// $g_c$'s linear bound for errors in `metric`, each input's and the +/// output's: $\beta_{c,i}$ and $b_{c,i}$ per input, read from the +/// operator and its arguments. `None` when the consumer has no error +/// contract for `metric`: an input whose error is in `metric` then gives +/// no edge (the metric rule). No factor is guessed. +pub fn error_propagation( + consumer: &Operator, + metric: AccuracyMetric, +) -> Option { + let _ = (consumer, metric); + todo!("#1207 PR 4: the error contract of each operator, per metric") +} + +/// The local error propagation of one consumer: its output error is at +/// most $\sum_i \beta_{c,i}\varepsilon_{c,i}$ while each input's error stays +/// within its validity cap (concepts doc, section 4; approved 2026-11-05). +#[derive(Debug, Clone, PartialEq)] +pub struct ErrorPropagation { + /// $\beta_{c,i}$ per input, for that input's metric. + pub factor_per_input: Vec, + /// $b_{c,i}$ per input: the cap under which the linear bound holds; + /// `None` when the bound holds everywhere (e.g. a ratio caps only its + /// denominator). + pub validity_cap_per_input: Vec>, +} diff --git a/crates/accuracy/src/estimators/mod.rs b/crates/accuracy/src/estimators/mod.rs deleted file mode 100644 index 6f5dd4f08..000000000 --- a/crates/accuracy/src/estimators/mod.rs +++ /dev/null @@ -1,730 +0,0 @@ -//! Dispatch committed estimator parameters to their accuracy models. -use super::*; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::FamilyCapability; - -pub(crate) mod cardinality; -pub(crate) mod cms; -pub(crate) mod count_sketch; -pub(crate) mod ddsketch; -pub mod hll; -pub(crate) mod kll; -pub mod univmon; - -pub(super) fn sketch_guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, -) -> Option { - match params { - SketchParams::Kll { .. } => kll::guarantee(algorithm, params, query), - SketchParams::DDSketch { .. } => ddsketch::guarantee(algorithm, params, query), - SketchParams::Hll { .. } => hll::generic_guarantee(algorithm, params, query), - SketchParams::Cms { .. } | SketchParams::CmsWithHeap { .. } => { - cms::guarantee(algorithm, params, query) - } - SketchParams::CountSketch { .. } | SketchParams::CountSketchWithHeap { .. } => { - count_sketch::guarantee(algorithm, params, query) - } - SketchParams::Kmv { .. } => cardinality::guarantee(algorithm, params, query), - SketchParams::UnivMon { .. } => univmon::guarantee(algorithm, params, query), - } -} - -fn bounded_guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, - metric: AccuracyMetric, - bound: f64, - delta: ProbabilityExpr, - contract: &str, -) -> AccuracyGuarantee { - AccuracyGuarantee::new( - metric, - BoundExpr::Constant { value: bound }, - delta, - vec![GuaranteeSource::SketchEvaluation { - algorithm: format!("{algorithm:?}"), - contract: contract.into(), - params: serde_json::to_value(params).unwrap_or(serde_json::Value::Null), - query: format!("{query:?}"), - }], - ) -} - -/// The built-in guarantee of reading `query` out of a summary of family -/// `family` built over an **exact** input, derived from the family's -/// committed parameters by inverting the same sizing formulas -/// [`size_params`] uses. Exact for a plain value -/// or an exact aggregate. `None` when a sketch does not answer `query` (its -/// family-capability table row), and when the family has no error model -/// (`SummarySample`, `Wavelet`, `StatModel`, and sketch evaluations without a -/// certified bound): Pass 1 then leaves the guarantee `None` and keeps the -/// candidate for plan selection to decide. -pub(crate) fn local_guarantee( - family: &FieldDataType, - query: &SketchStatistic, -) -> Option { - match family { - FieldDataType::Plain(_) => Some(AccuracyGuarantee::exact("Plain value")), - FieldDataType::ExactAggregate(kind, _) => Some(AccuracyGuarantee::exact(format!( - "ExactAggregate({kind:?})" - ))), - FieldDataType::Sketch(kind, _) if FamilyCapability::of(family).answers(query) => { - sketch_guarantee(kind.algorithm(), kind.params(), query) - } - FieldDataType::Sketch(..) => None, - // No error model is registered for these families. - FieldDataType::SummarySample(..) - | FieldDataType::Wavelet(..) - | FieldDataType::StatModel(..) => None, - } -} -/// `params` with the parameters its per-value failure probability depends -/// on enlarged, never reduced, so that each value it answers fails with -/// probability at most `delta` and its error bound is unchanged or smaller -/// (#669b): Count-Min's depth, Count Sketch's depth (the binomial tail of -/// its median), KLL's failure probability (its δ scale), a distinct count -/// UnivMon's heap, and KMV's `k` at the relative error it was sized for -/// (#694). Other parameters are returned as they are: DDSketch never fails; -/// classic HLL's failure probability comes from its source's distinct bound -/// ([`hll::min_precision`]); a KMV not -/// sized for a relative error and an L2 or entropy UnivMon are not -/// enlarged. -pub fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchParams { - if !(delta > 0.0 && delta < 1.0) { - return params.clone(); - } - match params { - SketchParams::Cms { width, depth } => SketchParams::Cms { - width: *width, - depth: (*depth).max(cms::cms_depth(delta)), - }, - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => SketchParams::CmsWithHeap { - width: *width, - depth: (*depth).max(cms::cms_depth(delta)), - heap_size: *heap_size, - }, - SketchParams::CountSketch { width, depth } => SketchParams::CountSketch { - width: *width, - depth: (*depth).max(count_sketch::count_sketch_depth(*width, delta)), - }, - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => SketchParams::CountSketchWithHeap { - width: *width, - depth: (*depth).max(count_sketch::count_sketch_depth(*width, delta)), - heap_size: *heap_size, - }, - SketchParams::Kll { .. } => kll::with_failure_probability(params, delta), - SketchParams::Kmv { - k, - relative_error: Some(eps), - } => SketchParams::Kmv { - k: (*k).max(cardinality::kmv_k(*eps, delta)), - relative_error: Some(*eps), - }, - SketchParams::UnivMon { .. } => univmon::with_failure_probability(params, delta), - _ => params.clone(), - } -} - -pub(crate) fn size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - match kind { - SketchAlgorithm::UnivMon => univmon::size_params(intent, eps, delta), - SketchAlgorithm::Kll => kll::size_params(eps, delta), - SketchAlgorithm::Cms => SketchParams::Cms { - width: cms::cms_width(eps), - depth: cms::cms_depth(delta), - }, - SketchAlgorithm::Hll => SketchParams::Hll { - precision: hll::hll_precision(eps, delta), - relative_error: Some(eps), - }, - SketchAlgorithm::CmsWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CmsWithHeap is only a TopK candidate"), - }; - SketchParams::CmsWithHeap { - width: cms::cms_width(eps), - depth: cms::cms_depth(delta), - heap_size: topk_capacity(k, eps), - } - } - // Non-preferred candidates (DDSketch / Kmv / CountSketch / - // CountSketchWithHeap) are only reachable once a cost model picks - // them; sized here so that wiring is local. - SketchAlgorithm::DDSketch => ddsketch::size_params(eps), - SketchAlgorithm::Kmv => SketchParams::Kmv { - k: cardinality::kmv_k(eps, delta), - relative_error: Some(eps), - }, - SketchAlgorithm::CountSketch => { - let width = count_sketch::count_sketch_width(eps); - SketchParams::CountSketch { - width, - depth: count_sketch::count_sketch_depth(width, delta), - } - } - SketchAlgorithm::CountSketchWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), - }; - let width = count_sketch::count_sketch_width(eps); - SketchParams::CountSketchWithHeap { - width, - depth: count_sketch::count_sketch_depth(width, delta), - heap_size: topk_capacity(k, eps), - } - } - // Listed, not `_`: a family `asap_sketchlib` adds fails to compile - // here until the planner sizes it or lists it as not offered. - SketchAlgorithm::CountSketchWithL2Norm - | SketchAlgorithm::CmsWithSpaceSaving - | SketchAlgorithm::CountSketchWithSpaceSaving - | SketchAlgorithm::CmsFold - | SketchAlgorithm::CountSketchFold - | SketchAlgorithm::HllHip - | SketchAlgorithm::KllDynamic - | SketchAlgorithm::SpaceSaving - | SketchAlgorithm::Elastic - | SketchAlgorithm::Coco - | SketchAlgorithm::Bloom - | SketchAlgorithm::SetAggregator - | SketchAlgorithm::UnivMonTwoTier - | SketchAlgorithm::UnivMonQ - | SketchAlgorithm::HydraCms - | SketchAlgorithm::HydraCountSketch - | SketchAlgorithm::HydraKll - | SketchAlgorithm::HydraHll - | SketchAlgorithm::HydraUnivMon => { - unreachable!("Pass 1 offers only the families in FAMILY_PREFERENCE, not {kind:?}") - } - } -} - -/// `P(Binomial(n, p) ≥ k)`, summed term by term in log space and rounded -/// up by a relative 1e-12 so rounding never certifies a smaller tail. -pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { - if k == 0 || p >= 1.0 { - return 1.0; - } - if k > n || p <= 0.0 { - return 0.0; - } - let (ln_p, ln_q) = (p.ln(), (-p).ln_1p()); - let mut ln_choose = 0.0; - let mut tail = 0.0; - for i in 0..=n { - if i > 0 { - ln_choose += f64::from(n - i + 1).ln() - f64::from(i).ln(); - } - if i >= k { - tail += (ln_choose + f64::from(i) * ln_p + f64::from(n - i) * ln_q).exp(); - } - } - (tail * (1.0 + 1e-12)).min(1.0) -} - -/// The parameters of one state shared by consumers of the same algorithm, -/// each sized alone with one of `params` (Pass 2's summary-capability -/// rule): each parameter's largest value, except where a smaller value is -/// stricter, where it takes the smallest (a KLL's failure probability, -/// DDSketch's α). Not every bound is monotone in its parameters (a -/// UnivMon's L2 collision bound is tighter at a power-of-two width than one -/// column wider), so the shared state is not assumed to meet each -/// consumer's target: the summary-capability rule composes every guarantee -/// above it again, and plan selection checks each consumer against the -/// shared state. `None` for no parameters, -/// parameters of different algorithms, or a shared UnivMon the library -/// would not build (`univmon::hash_independent`). -/// -/// A KLL's `k` and a distinct-count UnivMon's heap are the exceptions to -/// the largest value: their error grows as their failure probability -/// shrinks, so each reader's `k` or heap is first resized to the shared -/// (smallest) failure probability at no larger error (`kll::shared_k`, -/// `univmon::shared_heap`), and the shared build meets every reader. -pub fn size_for_all_readers(params: &[&SketchParams]) -> Option { - use SketchParams as P; - let (first, rest) = params.split_first()?; - let shared = rest.iter().try_fold((*first).clone(), |a, b| { - Some(match (a, *b) { - ( - P::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - distinct_failure_probability, - }, - P::UnivMon { - heap_size: h, - sketch_rows: r, - sketch_cols: c, - layers: l, - distinct_failure_probability: f, - }, - ) => P::UnivMon { - // Not the largest heap: a distinct count's error grows as - // its failure probability shrinks, so each reader's heap - // first grows to the shared one at no larger error. - heap_size: univmon::shared_heap(heap_size, distinct_failure_probability, *h, *f), - sketch_rows: sketch_rows.max(*r), - sketch_cols: sketch_cols.max(*c), - layers: layers.max(*l), - // `None`: not sized for a distinct count. - distinct_failure_probability: min_recorded(distinct_failure_probability, *f), - }, - ( - P::Kll { - k, - failure_probability, - }, - P::Kll { - k: k2, - failure_probability: f2, - }, - ) => P::Kll { - k: kll::shared_k(k, failure_probability, *k2, *f2), - // `None` is the calibration's δ, above any recorded one. - failure_probability: min_recorded(failure_probability, *f2), - }, - (P::Cms { width, depth }, P::Cms { width: w, depth: d }) => P::Cms { - width: width.max(*w), - depth: depth.max(*d), - }, - (P::CountSketch { width, depth }, P::CountSketch { width: w, depth: d }) => { - P::CountSketch { - width: width.max(*w), - depth: depth.max(*d), - } - } - ( - P::CmsWithHeap { - width, - depth, - heap_size, - }, - P::CmsWithHeap { - width: w, - depth: d, - heap_size: h, - }, - ) => P::CmsWithHeap { - width: width.max(*w), - depth: depth.max(*d), - heap_size: heap_size.max(*h), - }, - ( - P::CountSketchWithHeap { - width, - depth, - heap_size, - }, - P::CountSketchWithHeap { - width: w, - depth: d, - heap_size: h, - }, - ) => P::CountSketchWithHeap { - width: width.max(*w), - depth: depth.max(*d), - heap_size: heap_size.max(*h), - }, - ( - P::Hll { - precision, - relative_error, - }, - P::Hll { - precision: p, - relative_error: e, - }, - ) => P::Hll { - precision: precision.max(*p), - relative_error: min_recorded(relative_error, *e), - }, - (P::DDSketch { alpha }, P::DDSketch { alpha: a }) => P::DDSketch { - alpha: alpha.min(*a), - }, - ( - P::Kmv { k, relative_error }, - P::Kmv { - k: k2, - relative_error: e, - }, - ) => P::Kmv { - k: k.max(*k2), - relative_error: min_recorded(relative_error, *e), - }, - _ => return None, - }) - })?; - // The largest rows of one UnivMon with the largest columns of another - // can pass the hash budget `asap_sketchlib` builds within. - match shared { - P::UnivMon { - sketch_rows, - sketch_cols, - layers, - .. - } if !univmon::hash_independent(sketch_rows, sketch_cols, layers) => None, - shared => Some(shared), - } -} - -/// The smaller of two recorded failure probabilities; an unrecorded one -/// (`None`) yields to a recorded one. -fn min_recorded(a: Option, b: Option) -> Option { - match (a, b) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - } -} - -/// Accuracy-dependent candidate budget, not a completeness theorem. The -/// membership model must still certify the selected set independently. -pub(crate) fn topk_capacity(k: usize, eps: f64) -> u32 { - u32::try_from(k) - .unwrap_or(u32::MAX) - .max(saturating_ceil(1.0 / eps, 1, 1 << 26)) -} - -/// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive x saturate to `hi` -/// (a degenerate ε means "as accurate as this family goes"). -pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { - if !x.is_finite() || x <= 0.0 { - return hi; - } - (x.ceil() as u32).clamp(lo, hi) -} - -#[cfg(test)] -mod shared_size_tests { - use super::*; - - /// The shared size takes each parameter's largest value, and the - /// smallest where smaller is stricter (KLL δ, DDSketch α); sizes of - /// different algorithms have none. - #[test] - fn shared_size_takes_each_parameters_strictest_value() { - assert_eq!( - size_for_all_readers(&[&univmon(256, 5, 1024, 16), &univmon(1, 7, 4096, 1)]), - Some(univmon(256, 7, 4096, 16)) - ); - // The distinct count's failure probability, where one was sized - // for, takes the smallest. - let distinct = |heap_size, delta| SketchParams::UnivMon { - heap_size, - sketch_rows: 1, - sketch_cols: 1, - layers: 19, - distinct_failure_probability: Some(delta), - }; - let Some(SketchParams::UnivMon { - heap_size, - distinct_failure_probability, - .. - }) = size_for_all_readers(&[ - &univmon(256, 5, 1024, 16), - &distinct(4096, 0.01), - &distinct(2048, 0.001), - ]) - else { - unreachable!() - }; - // The heap of 4096 sized at δ = 0.01 grows to the shared δ = 0.001 - // at its own relative error. - assert_eq!( - (heap_size, distinct_failure_probability), - (40_961, Some(0.001)) - ); - let kll = |k, failure_probability| SketchParams::Kll { - k, - failure_probability, - }; - let SketchParams::Kll { - k, - failure_probability, - } = size_for_all_readers(&[ - &kll(200, None), - &kll(100, Some(0.001)), - &kll(150, Some(0.005)), - ]) - .unwrap() - else { - unreachable!() - }; - // The `k = 200` reader, sized at the calibration's δ = 0.01, is - // resized to the shared δ = 0.001 at its own rank error. - assert_eq!((k, failure_probability), (258, Some(0.001))); - assert_eq!( - size_for_all_readers(&[ - &SketchParams::DDSketch { alpha: 0.01 }, - &SketchParams::DDSketch { alpha: 0.001 } - ]), - Some(SketchParams::DDSketch { alpha: 0.001 }) - ); - assert_eq!( - size_for_all_readers(&[ - &kll(200, None), - &SketchParams::Hll { - precision: 12, - relative_error: None - } - ]), - None - ); - assert_eq!(size_for_all_readers(&[]), None); - } - - /// A KLL shared by a reader at rank (0.01, δ = 0.01) and one at - /// (0.05, δ = 1e-4) meets both: each reader's `k` is resized to the - /// shared δ = 1e-4 at no larger rank error (A's 0.00997) before the - /// larger is taken (k = 412), not 269 stated at 1e-4 (rank error 0.0151). - #[test] - fn a_kll_shared_across_failure_probabilities_meets_each_reader() { - let readers = [(0.01, 0.01), (0.05, 1e-4)]; - let params = readers.map(|(eps, delta)| kll::size_params(eps, delta)); - let shared = size_for_all_readers(&[¶ms[0], ¶ms[1]]).unwrap(); - let quantile = SketchStatistic::Quantile { q: 0.99 }; - let guarantee = sketch_guarantee(&SketchAlgorithm::Kll, &shared, &quantile).unwrap(); - for (eps, delta) in readers { - assert!( - satisfies( - &guarantee, - &AccuracyTarget::max_error(eps) - .failure_probability(delta) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - ), - "{shared:?} misses ({eps}, {delta})" - ); - } - assert_eq!( - shared, - SketchParams::Kll { - k: 412, - failure_probability: Some(1e-4) - } - ); - } - - fn univmon(heap_size: u32, sketch_rows: u32, sketch_cols: u32, layers: u8) -> SketchParams { - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - distinct_failure_probability: None, - } - } - - /// Each UnivMon fits `asap_sketchlib`'s hash budget, but the largest - /// rows of one with the largest columns of the other do not - /// (`12 · (⌈log2 2^16⌉ + 1) = 204 > 128`): the library would not build - /// the shared shape, so there is none. - #[test] - fn no_shared_univmon_past_the_hash_budget() { - assert!(univmon::hash_independent(12, 512, 1)); - assert!(univmon::hash_independent(5, 1 << 16, 1)); - assert_eq!( - size_for_all_readers(&[&univmon(1, 12, 512, 1), &univmon(1, 5, 1 << 16, 1)]), - None - ); - } - - /// A distinct-count UnivMon shared by a reader at relative (0.05, - /// δ = 0.01) and one at (0.2, δ = 1e-4) meets both: each reader's heap - /// grows to the shared δ = 1e-4 at no larger error before the larger is - /// taken (5 400 000), not B's 337 500 stated at 1e-4 (error 0.2). - #[test] - fn a_distinct_univmon_shared_across_failure_probabilities_meets_each_reader() { - use asap_types::ir::operator::agg_intent::default_cardinality; - let readers = [(0.05, 0.01), (0.2, 1e-4)]; - let params = readers.map(|(eps, delta)| { - size_params(SketchAlgorithm::UnivMon, &default_cardinality(), eps, delta) - }); - let shared = size_for_all_readers(&[¶ms[0], ¶ms[1]]).unwrap(); - let guarantee = univmon::distinct_guarantee(&shared, "m").unwrap(); - let error = guarantee.spec.error.evaluate().unwrap(); - // The top-layer term stays symbolic in the source's distinct bound; - // the Chebyshev term is the share of δ the heap was sized for. - let ProbabilityExpr::UnionBound { terms } = &guarantee.spec.failure_probability else { - panic!("{guarantee:?}") - }; - let chebyshev = terms[0].evaluate().unwrap(); - for (eps, delta) in readers { - assert!( - error <= eps && chebyshev <= delta, - "{shared:?} misses ({eps}, {delta}): error {error}" - ); - } - let SketchParams::UnivMon { - heap_size, - distinct_failure_probability, - .. - } = shared - else { - unreachable!() - }; - assert_eq!( - (heap_size, distinct_failure_probability), - (5_400_000, Some(1e-4)) - ); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile}; - - fn delta_and_error(params: &SketchParams, query: &SketchStatistic) -> (f64, f64) { - let algorithm = match params { - SketchParams::Cms { .. } => SketchAlgorithm::Cms, - SketchParams::CountSketch { .. } => SketchAlgorithm::CountSketch, - SketchParams::Kll { .. } => SketchAlgorithm::Kll, - other => panic!("{other:?}"), - }; - let g = sketch_guarantee(&algorithm, params, query).unwrap(); - ( - g.spec.failure_probability.evaluate().unwrap(), - g.spec.error.evaluate().unwrap(), - ) - } - - /// Enlarging for a smaller failure probability deepens Count-Min and - /// Count Sketch and grows KLL until each meets it, at no larger error; - /// it never shrinks a sketch that already meets it. (Count Sketch's - /// depth is capped at 255 rows, which a non-power-of-two width reaches - /// before 1e-6.) - #[test] - fn enlarging_meets_the_failure_probability_at_no_larger_error() { - let point = SketchStatistic::PointCount { - key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, - value: None, - }; - let quantile = SketchStatistic::Quantile { q: 0.99 }; - for (params, query, delta) in [ - ( - size_params(SketchAlgorithm::Cms, &default_cardinality(), 0.01, 0.01), - &point, - 1e-6, - ), - ( - size_params( - SketchAlgorithm::CountSketch, - &default_cardinality(), - 0.01, - 0.01, - ), - &point, - 1e-3, - ), - ( - size_params(SketchAlgorithm::Kll, &default_quantile(0.99), 0.01, 0.01), - &quantile, - 1e-6, - ), - ] { - let (_, error) = delta_and_error(¶ms, query); - let enlarged = with_failure_probability(¶ms, delta); - let (new_delta, new_error) = delta_and_error(&enlarged, query); - assert!(new_delta <= delta, "{enlarged:?}"); - assert!(new_error <= error * (1.0 + 1e-9), "{enlarged:?}"); - assert_ne!(enlarged, params); - assert_eq!(with_failure_probability(&enlarged, 0.01), enlarged); - } - let ddsketch = SketchParams::DDSketch { alpha: 0.01 }; - assert_eq!(with_failure_probability(&ddsketch, 1e-6), ddsketch); - } - - /// A distinct-count UnivMon grows its heap for a smaller failure - /// probability at no larger relative error. - #[test] - fn a_distinct_count_univmon_grows_its_heap() { - let params = size_params(SketchAlgorithm::UnivMon, &default_cardinality(), 0.05, 0.01); - let enlarged = with_failure_probability(¶ms, 1e-4); - let error = |p: &SketchParams| { - univmon::distinct_guarantee(p, "m") - .unwrap() - .spec - .error - .evaluate() - .unwrap() - }; - let ( - SketchParams::UnivMon { heap_size: k, .. }, - SketchParams::UnivMon { - heap_size: grown, - distinct_failure_probability, - .. - }, - ) = (¶ms, &enlarged) - else { - panic!("{enlarged:?}") - }; - assert!(grown > k); - assert_eq!(*distinct_failure_probability, Some(1e-4)); - assert!(error(&enlarged) <= error(¶ms) * (1.0 + 1e-9)); - } - - /// Classic HLL's precision grows until its failure probability over the - /// source's distinct bound meets the target, up to 18. - #[test] - fn classic_hll_precision_grows_for_its_failure_probability() { - let p = hll::min_precision(0.05, 0.01, 64, 4); - assert!(p > 4); - assert!(hll::classic_failure_probability(p, 0.05, 64) <= 0.01); - assert!(hll::classic_failure_probability(p - 1, 0.05, 64) > 0.01); - assert_eq!(hll::min_precision(0.05, 1e-30, 4096, 12), 12); - } - - /// A sketch read for a statistic its table row does not answer (an HLL - /// for a total) has no guarantee; one it answers keeps its guarantee, - /// and a plain value or exact aggregate stays exact. - #[test] - fn local_guarantee_needs_a_statistic_the_family_answers() { - use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::ir::schema::{ - DataType, ExactKind, ExactParams, GroupingStrategy, SketchKind, - }; - let hll = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, - ), - GroupingStrategy::default(), - ); - let total = SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }; - assert!(local_guarantee(&hll, &total).is_none()); - assert!(local_guarantee(&hll, &SketchStatistic::Cardinality).is_some()); - for exact in [ - FieldDataType::Plain(DataType::Float64), - FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), - ] { - assert!(local_guarantee(&exact, &total).is_some(), "{exact:?}"); - } - } -} diff --git a/crates/accuracy/src/evidence.rs b/crates/accuracy/src/evidence.rs index 7a66c62ea..19274b6fb 100644 --- a/crates/accuracy/src/evidence.rs +++ b/crates/accuracy/src/evidence.rs @@ -1,51 +1,42 @@ -//! Scoped source contracts and statistics the accuracy rules consult. -use super::*; +//! The deployment's measured accuracy, read before each summary's +//! theoretical guarantee. -/// An enforced domain for the true values of one operand of an exact -/// arithmetic operation, in every evaluation: they are finite and lie in -/// `[lower, upper]`. Plan selection uses it to bound a product or quotient -/// of estimates whose errors are not both relative. Sampled min/max -/// statistics are not a proof: the source or execution layer must enforce -/// it. -#[derive(Debug, Clone, PartialEq)] -pub struct OperandDomain { - pub lower: f64, - pub upper: f64, +use asap_types::accuracy::AccuracyGuarantee; +use asap_types::ir::schema::FieldDataType; +use asap_types::summary_type::{SketchParams, SketchStatistic}; + +/// The deployment's measured accuracy, per summary configuration. A +/// producer's guarantee for a readout is its measurement when there is one, +/// else [`theoretical_guarantee`](crate::guarantee::theoretical_guarantee); +/// the source is decided before Pass 2 builds the logical edges (concepts +/// doc, section 2, "Where the guarantee comes from"). +pub trait AccuracyEvidence { + /// The measured error bound of reading `statistic` from a state of + /// `summary_type` with parameters `parameters` at failure probability + /// `failure_probability` (the guarantee's own failure probability); + /// `None` when the deployment has no measurement for it, or when this θ + /// cannot fail that rarely. + fn measured_guarantee( + &self, + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, + failure_probability: f64, + ) -> Option; } -/// Statistics a propagation rule may consult. Every field is optional and -/// defaults to "unknown": a rule that needs a missing statistic emits a -/// [`BoundExpr::Unknown`] leaf (or rejects) rather than guessing. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct PropagationStats { - /// Provenance for supplied evidence (source, observation identity, etc.). - pub(crate) evidence_provenance: Vec, - /// Whether every input value is known to be non-negative — required by - /// the multiplicative relative-error rule, which is unsound across a - /// sign change. - pub(crate) values_non_negative: Option, - /// The statistic an unknown sign is named by when the folded values are - /// one metric's raw data samples or plain sums of them - /// ([`raw_data_samples_sign_statistic`](crate::composition::raw_data_samples_sign_statistic)): - /// the relative bound is then gated on it, and plan selection binds it - /// to 1 when the metric is declared a counter. `None`: the bare - /// `values_non_negative`, which nothing binds. - pub(crate) sign_statistic: Option, - /// Number of input rows an exact aggregation consumes (e.g. the number - /// of groups a function folds), for exact aggregate union bounds - /// bound over per-input failures. - pub(crate) input_row_count: Option, - /// The statistic an unknown row count is named by (default - /// `input_row_count`), so a fold whose rows the data workload's input - /// cardinality does not bound can leave a statistic nothing binds. - pub(crate) row_count_statistic: Option<&'static str>, - /// Lower confidence bound of the kth selected TopK item, after widening - /// the interval by the sketch's own estimation error. - pub(crate) topk_selected_lower_bound: Option, - /// Greatest upper confidence bound among excluded TopK items, after - /// widening the interval by the sketch's own estimation error. - pub(crate) topk_excluded_upper_bound: Option, - /// Union-bound failure probability of all intervals used by the margin - /// certificate. - pub(crate) topk_interval_failure_probability: Option, +/// Evidence with no measurement: every guarantee is the theoretical one. +#[derive(Debug, Clone, Copy)] +pub struct NoMeasuredGuarantees; + +impl AccuracyEvidence for NoMeasuredGuarantees { + fn measured_guarantee( + &self, + _summary_type: &FieldDataType, + _parameters: &SketchParams, + _statistic: &SketchStatistic, + _failure_probability: f64, + ) -> Option { + None + } } diff --git a/crates/accuracy/src/function_rules.rs b/crates/accuracy/src/function_rules.rs index fd5eb81d4..1a91edea8 100644 --- a/crates/accuracy/src/function_rules.rs +++ b/crates/accuracy/src/function_rules.rs @@ -1,8 +1,8 @@ //! The accuracy composition rule of each exact function applied to an //! approximate input, shared by Pass 1's guarantee composition and the //! accuracy algebra. +use asap_types::accuracy::CompositionOperator; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::CompositionOperator; /// The composition operator that propagates an approximate input's guarantee /// through `intent`; `None` for a function with no registered rule over diff --git a/crates/accuracy/src/estimators/cardinality.rs b/crates/accuracy/src/guarantee/cardinality.rs similarity index 67% rename from crates/accuracy/src/estimators/cardinality.rs rename to crates/accuracy/src/guarantee/cardinality.rs index 8d6217911..a437da6de 100644 --- a/crates/accuracy/src/estimators/cardinality.rs +++ b/crates/accuracy/src/guarantee/cardinality.rs @@ -41,10 +41,9 @@ pub(super) fn guarantee( query: &SketchStatistic, ) -> Option { match params { - SketchParams::Kmv { k, relative_error } => { - // Not sized for a target: its relative standard error. - let eps = relative_error - .unwrap_or_else(|| 1.0 / f64::from(k.saturating_sub(2).max(1)).sqrt()); + SketchParams::Kmv { k } => { + // Its relative standard error. + let eps = 1.0 / f64::from(k.saturating_sub(2).max(1)).sqrt(); Some(super::bounded_guarantee( algorithm, params, @@ -113,61 +112,6 @@ pub(crate) fn kmv_k(eps: f64, delta: f64) -> u32 { #[cfg(test)] mod tests { use super::*; - use asap_types::ir::operator::agg_intent::default_cardinality; - - fn error_and_delta(params: &SketchParams) -> (f64, f64) { - let g = guarantee(&SketchAlgorithm::Kmv, params, &SketchStatistic::Cardinality).unwrap(); - ( - g.spec.error.evaluate().unwrap(), - g.spec.failure_probability.evaluate().unwrap(), - ) - } - - /// KMV sized for (ε, δ) certifies both (#700): its failure probability - /// is the requested δ's, not a fixed 0.01. - #[test] - fn kmv_sized_for_a_failure_probability_meets_it() { - for (eps, delta) in [(0.01, 0.001), (0.05, 0.01), (0.02, 1e-6)] { - let params = - super::super::size_params(SketchAlgorithm::Kmv, &default_cardinality(), eps, delta); - let (error, failure_probability) = error_and_delta(¶ms); - assert!(error <= eps, "{params:?}: {error}"); - assert!( - failure_probability <= delta, - "{params:?}: {failure_probability}" - ); - } - } - - /// KMV's sizing grows `k` for a smaller δ at the same ε, near the - /// module's closed form, and a KMV sized for one δ is enlarged for a - /// smaller one at the same relative error. - #[test] - fn kmv_k_follows_the_failure_probability() { - // Ten times smaller than the former Chebyshev calibration's 40,002. - assert_eq!(kmv_k(0.05, 0.01), 4323); - assert_eq!(kmv_k(0.02, 0.01), 26656); - assert!(kmv_k(0.05, 1e-6) > kmv_k(0.05, 0.01)); - assert!(kmv_failure_probability(kmv_k(0.05, 0.01) - 1, 0.05) > 0.01); - assert_eq!(kmv_k(0.0, 0.01), KMV_MAX_K); - let sized = - super::super::size_params(SketchAlgorithm::Kmv, &default_cardinality(), 0.05, 0.01); - let enlarged = super::super::with_failure_probability(&sized, 1e-4); - let (error, delta) = error_and_delta(&enlarged); - assert_eq!(error, 0.05); - assert!(delta <= 1e-4); - assert_ne!(enlarged, sized); - // Not sized for a relative error: nothing to enlarge for. - let unsized_kmv = SketchParams::Kmv { - k: 1024, - relative_error: None, - }; - assert_eq!( - super::super::with_failure_probability(&unsized_kmv, 1e-4), - unsized_kmv - ); - assert_eq!(error_and_delta(&unsized_kmv).1, 1.0); - } /// The bound dominates the exact failure probability of the order /// statistic for every number of distinct items checked: exact below diff --git a/crates/accuracy/src/estimators/cms.rs b/crates/accuracy/src/guarantee/cms.rs similarity index 70% rename from crates/accuracy/src/estimators/cms.rs rename to crates/accuracy/src/guarantee/cms.rs index 663749928..6895ac6e1 100644 --- a/crates/accuracy/src/estimators/cms.rs +++ b/crates/accuracy/src/guarantee/cms.rs @@ -41,15 +41,12 @@ mod tests { #[test] fn local_guarantee_inverts_frequency_sizing() { - use crate::estimators::size_params; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; + use crate::guarantee::size_params; let c = asap_types::ir::operator::agg_intent::default_cardinality(); let params = size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); - let g = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Cms, params), - GroupingStrategy::default(), - ), + let g = sketch_guarantee( + &SketchAlgorithm::Cms, + ¶ms, &SketchStatistic::PointCount { key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, value: None, @@ -57,28 +54,19 @@ mod tests { ) .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1); - assert!(satisfies( - &g, - &AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - )); + assert!(meets(&g, 0.01, Some(0.001))); } #[test] fn heap_evaluation_retains_frequency_metric() { - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let cms_heap = SketchParams::CmsWithHeap { width: 272, depth: 5, heap_size: 10, }; - let topk_frequency = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::CmsWithHeap, cms_heap), - GroupingStrategy::default(), - ), + let topk_frequency = sketch_guarantee( + &SketchAlgorithm::CmsWithHeap, + &cms_heap, &SketchStatistic::TopK { k: 10 }, ) .expect("heap sketch still provides per-key frequency intervals"); @@ -91,11 +79,8 @@ mod tests { /// A top-k heap holds `max(k, ⌈1/ε⌉)` entries over a CMS sized as for counts. #[test] fn topk_heap_capacity_respects_accuracy_and_output_count() { - use crate::estimators::size_params; - let intent = AggIntent::TopK { - k: 25, - accuracy: AccuracyTarget::max_error(0.01), - }; + use crate::guarantee::size_params; + let intent = AggIntent::TopK { k: 25 }; assert_eq!( size_params(SketchAlgorithm::CmsWithHeap, &intent, 0.01, 0.01), SketchParams::CmsWithHeap { diff --git a/crates/accuracy/src/estimators/count_sketch.rs b/crates/accuracy/src/guarantee/count_sketch.rs similarity index 92% rename from crates/accuracy/src/estimators/count_sketch.rs rename to crates/accuracy/src/guarantee/count_sketch.rs index dac391fe0..4f72dfd99 100644 --- a/crates/accuracy/src/estimators/count_sketch.rs +++ b/crates/accuracy/src/guarantee/count_sketch.rs @@ -155,16 +155,13 @@ mod tests { #[test] fn count_sketch_uses_an_l2_guarantee() { - use crate::estimators::size_params; + use crate::guarantee::size_params; use asap_types::ir::operator::agg_intent::default_cardinality; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let intent = default_cardinality(); let count_sketch = size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); - let guarantee = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::CountSketch, count_sketch), - GroupingStrategy::default(), - ), + let guarantee = sketch_guarantee( + &SketchAlgorithm::CountSketch, + &count_sketch, &SketchStatistic::PointCount { key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, value: Some("1".into()), @@ -175,13 +172,7 @@ mod tests { guarantee.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L2 ); - assert!(satisfies( - &guarantee, - &AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L2) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - )); + assert!(meets(&guarantee, 0.01, Some(0.01))); } /// The depth is the smallest odd one whose binomial tail meets δ. Each diff --git a/crates/accuracy/src/estimators/ddsketch.rs b/crates/accuracy/src/guarantee/ddsketch.rs similarity index 100% rename from crates/accuracy/src/estimators/ddsketch.rs rename to crates/accuracy/src/guarantee/ddsketch.rs diff --git a/crates/accuracy/src/estimators/hll.rs b/crates/accuracy/src/guarantee/hll.rs similarity index 95% rename from crates/accuracy/src/estimators/hll.rs rename to crates/accuracy/src/guarantee/hll.rs index 18d45db02..7da74066d 100644 --- a/crates/accuracy/src/estimators/hll.rs +++ b/crates/accuracy/src/guarantee/hll.rs @@ -81,6 +81,7 @@ pub fn parse_classic_failure_statistic(statistic: &str) -> Option<(u8, f64, &str /// distinct bound ([`classic_failure_statistic`]). The failure probability /// is per output row: each group's estimate fails with at most it. `None` /// for a precision outside the contract's. +#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. pub(crate) fn classic_guarantee( precision: u8, relative_error: Option, @@ -460,47 +461,32 @@ mod tests { /// bound. #[test] fn precision_is_sized_from_epsilon_and_delta() { - use crate::estimators::size_params; + use crate::guarantee::size_params; use asap_types::ir::operator::agg_intent::default_cardinality; for (eps, delta, precision) in [(0.05, 0.01, 15), (0.02, 0.01, 17)] { let params = size_params(SketchAlgorithm::Hll, &default_cardinality(), eps, delta); - let SketchParams::Hll { - precision: sized, - relative_error, - } = params - else { + let SketchParams::Hll { precision: sized } = params else { panic!("{params:?}") }; - assert_eq!((sized, relative_error), (precision, Some(eps))); + assert_eq!(sized, precision); } } #[test] fn generic_rse_sizing_does_not_certify_confidence() { - use crate::estimators::size_params; + use crate::guarantee::size_params; use asap_types::ir::operator::agg_intent::default_cardinality; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = default_cardinality(); let params = size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); - let g = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Hll, params), - GroupingStrategy::default(), - ), + let g = sketch_guarantee( + &SketchAlgorithm::Hll, + ¶ms, &SketchStatistic::Cardinality, ) .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative); assert_eq!(g.spec.failure_probability.evaluate(), None); - assert!(satisfies( - &g, - &AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative) - )); - assert!(!satisfies( - &g, - &AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Relative) - )); + assert!(meets(&g, 0.01, None)); + assert!(!meets(&g, 0.01, Some(0.01))); } } diff --git a/crates/accuracy/src/estimators/kll.rs b/crates/accuracy/src/guarantee/kll.rs similarity index 57% rename from crates/accuracy/src/estimators/kll.rs rename to crates/accuracy/src/guarantee/kll.rs index 92bee9d84..bc6f0f010 100644 --- a/crates/accuracy/src/estimators/kll.rs +++ b/crates/accuracy/src/guarantee/kll.rs @@ -2,24 +2,20 @@ //! and, below a 0.01 failure probability, that calibration scaled by the //! Gaussian quantile ratio `z(1 − δ/2) / z(1 − 0.01/2)`. use super::*; -use asap_types::ir::schema::sketch::KLL_MAX_K; +use asap_types::summary_type::KLL_MAX_K; pub(super) fn guarantee( algorithm: &SketchAlgorithm, params: &SketchParams, query: &SketchStatistic, ) -> Option { - let SketchParams::Kll { - k, - failure_probability, - } = params - else { + let SketchParams::Kll { k } = params else { return None; }; if *k > KLL_MAX_K { return None; } - let delta = failure_probability.unwrap_or(KLL_CALIBRATED_FAILURE_PROBABILITY); + let delta = KLL_CALIBRATED_FAILURE_PROBABILITY; let scale = delta_scale(delta)?; let contract = if delta < KLL_CALIBRATED_FAILURE_PROBABILITY { KLL_DELTA_SCALED_CONTRACT @@ -139,73 +135,11 @@ fn normal_quantile(p: f64) -> f64 { /// `eps / delta_scale(delta)`, recording `delta` so the guarantee reports it. pub(crate) fn size_params(eps: f64, delta: f64) -> SketchParams { if delta.is_nan() || delta <= 0.0 || delta >= KLL_CALIBRATED_FAILURE_PROBABILITY { - return SketchParams::Kll { - k: kll_k(eps), - failure_probability: None, - }; + return SketchParams::Kll { k: kll_k(eps) }; } SketchParams::Kll { k: kll_k(eps / delta_scale(delta).unwrap_or(1.0)), - failure_probability: Some(delta), - } -} - -/// `params` resized to fail with probability at most `delta` at no larger -/// rank error: `k` for its current error at `delta`'s scale, never smaller. -/// Unchanged when it already fails no more often. -pub(crate) fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchParams { - let SketchParams::Kll { - k, - failure_probability, - } = params - else { - return params.clone(); - }; - let current = failure_probability.unwrap_or(KLL_CALIBRATED_FAILURE_PROBABILITY); - let Some(scale) = delta_scale(current) else { - return params.clone(); - }; - if delta >= current { - return params.clone(); } - match size_params(kll_rank_error_99(*k) * scale, delta) { - SketchParams::Kll { - k: resized, - failure_probability, - } => SketchParams::Kll { - k: resized.max(*k), - failure_probability, - }, - other => other, - } -} - -/// The `k` of one KLL shared by two readers sized alone at `(k, δ)` and -/// `(k2, δ2)`: each `k` resized to the smaller failure probability at no -/// larger rank error ([`with_failure_probability`]), then the larger. The -/// shared build states its guarantee at that smaller δ, where a `k` sized -/// for a larger δ has a larger rank error, so the larger `k` alone could -/// miss its reader's ε. -pub(crate) fn shared_k( - k: u32, - failure_probability: Option, - k2: u32, - failure_probability2: Option, -) -> u32 { - let delta = failure_probability - .unwrap_or(KLL_CALIBRATED_FAILURE_PROBABILITY) - .min(failure_probability2.unwrap_or(KLL_CALIBRATED_FAILURE_PROBABILITY)); - let resized = |k, failure_probability| match with_failure_probability( - &SketchParams::Kll { - k, - failure_probability, - }, - delta, - ) { - SketchParams::Kll { k, .. } => k, - _ => k, - }; - resized(k, failure_probability).max(resized(k2, failure_probability2)) } #[cfg(test)] @@ -214,32 +148,20 @@ mod tests { #[test] fn local_guarantee_inverts_rank_sizing() { - use crate::estimators::size_params; + use crate::guarantee::size_params; use asap_types::ir::operator::agg_intent::default_quantile; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let q = default_quantile(0.99); let params = size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01); - let g = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, params), - GroupingStrategy::default(), - ), + let g = sketch_guarantee( + &SketchAlgorithm::Kll, + ¶ms, &SketchStatistic::Quantile { q: 0.99 }, ) .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Rank); - assert!(satisfies( - &g, - &AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank) - )); + assert!(meets(&g, 0.01, None)); assert_eq!(g.spec.failure_probability.evaluate(), Some(0.01)); - assert!(satisfies( - &g, - &AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - )); + assert!(meets(&g, 0.01, Some(0.01))); assert_eq!(g.approximate_layer_count(), 1); assert!(g.provenance.iter().any(|source| matches!( source, @@ -248,61 +170,6 @@ mod tests { ))); } - /// Below δ = 0.01, KLL is sized by the KLL bound's δ dependence: k is - /// larger than at δ = 0.01, and the guarantee meets ε = 0.01, δ = 0.001. - #[test] - fn sizing_meets_a_failure_probability_below_one_percent() { - use crate::estimators::size_params; - use asap_types::ir::operator::agg_intent::default_quantile; - let q = default_quantile(0.99); - let k = |delta| match size_params(SketchAlgorithm::Kll, &q, 0.01, delta) { - SketchParams::Kll { k, .. } => k, - other => panic!("{other:?}"), - }; - assert!(k(0.001) > k(0.01), "{} vs {}", k(0.001), k(0.01)); - let params = size_params(SketchAlgorithm::Kll, &q, 0.01, 0.001); - let g = guarantee( - &SketchAlgorithm::Kll, - ¶ms, - &SketchStatistic::Quantile { q: 0.99 }, - ) - .unwrap(); - assert!(satisfies( - &g, - &AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - )); - assert_eq!(g.spec.failure_probability.evaluate(), Some(0.001)); - } - - /// A KLL sized below δ = 0.01 names its own contract, not the empirical - /// 99% calibration's. - #[test] - fn sizing_below_one_percent_names_a_distinct_contract() { - let contract = |delta| { - let g = guarantee( - &SketchAlgorithm::Kll, - &size_params(0.01, delta), - &SketchStatistic::Quantile { q: 0.99 }, - ) - .unwrap(); - match &g.provenance[..] { - [GuaranteeSource::SketchEvaluation { contract, .. }] => contract.clone(), - other => panic!("{other:?}"), - } - }; - assert_eq!( - contract(0.01), - "apache_datasketches_kll_empirical_99_a9b42755072b" - ); - assert_eq!( - contract(0.001), - "apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b" - ); - } - /// `k` never exceeds the 26 602 `asap_sketchlib` silently caps it at, so /// a guarantee never claims a larger `k` than runs. #[test] @@ -317,10 +184,7 @@ mod tests { let g = |k| { guarantee( &SketchAlgorithm::Kll, - &SketchParams::Kll { - k, - failure_probability: None, - }, + &SketchParams::Kll { k }, &SketchStatistic::Quantile { q: 0.99 }, ) }; @@ -350,65 +214,34 @@ mod tests { size_params(0.01, 0.001), SketchParams::Kll { k: kll_k(0.01 / ratio), - failure_probability: Some(0.001), } ); } - /// A hand-built failure probability outside (0, 1) gives no guarantee - /// rather than an infinite or NaN error bound. - #[test] - fn an_invalid_failure_probability_gives_no_guarantee() { - for delta in [0.0, -0.1, 1.0, 1.5, f64::NAN, f64::INFINITY] { - let params = SketchParams::Kll { - k: 200, - failure_probability: Some(delta), - }; - assert!( - guarantee( - &SketchAlgorithm::Kll, - ¶ms, - &SketchStatistic::Quantile { q: 0.99 }, - ) - .is_none(), - "{delta}" - ); - } - } - /// KLL `k` inverts the rank-error fit: ε = 0.01 → 269, ε = 0.05 → 52. #[test] fn accuracy_target_drives_kll_k() { - use crate::estimators::size_params; + use crate::guarantee::size_params; use asap_types::ir::operator::agg_intent::default_quantile; let q = default_quantile(0.99); assert_eq!( size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01), - SketchParams::Kll { - k: 269, - failure_probability: None - } + SketchParams::Kll { k: 269 } ); assert_eq!( size_params(SketchAlgorithm::Kll, &q, 0.05, 0.01), - SketchParams::Kll { - k: 52, - failure_probability: None - } + SketchParams::Kll { k: 52 } ); } /// A zero ε saturates KLL `k` to the tightest buildable value. #[test] fn degenerate_epsilon_saturates_to_tightest_kll_params() { - use crate::estimators::size_params; + use crate::guarantee::size_params; use asap_types::ir::operator::agg_intent::default_quantile; assert_eq!( size_params(SketchAlgorithm::Kll, &default_quantile(0.99), 0.0, 0.01), - SketchParams::Kll { - k: 26_602, - failure_probability: None - } + SketchParams::Kll { k: 26_602 } ); } } diff --git a/crates/accuracy/src/guarantee/mod.rs b/crates/accuracy/src/guarantee/mod.rs new file mode 100644 index 000000000..9769da79f --- /dev/null +++ b/crates/accuracy/src/guarantee/mod.rs @@ -0,0 +1,224 @@ +//! Dispatch committed estimator parameters to their accuracy models. +use asap_types::accuracy::{ + AccuracyGuarantee, AccuracyMetric, BoundExpr, GuaranteeSource, ProbabilityExpr, +}; +use asap_types::ir::schema::FieldDataType; +use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; + +#[cfg(test)] +use crate::requirements::meets; +use asap_types::ir::operator::AggIntent; +use asap_types::summary_type::SummaryTypeCapability; + +pub(crate) mod cardinality; +pub(crate) mod cms; +pub(crate) mod count_sketch; +pub(crate) mod ddsketch; +pub mod hll; +pub(crate) mod kll; +pub mod univmon; + +#[allow(dead_code)] // Read again for the error bound of each listed θ (`parameter_list`), in #1207 PRs 3–4. +pub(super) fn sketch_guarantee( + algorithm: &SketchAlgorithm, + params: &SketchParams, + query: &SketchStatistic, +) -> Option { + match params { + SketchParams::Kll { .. } => kll::guarantee(algorithm, params, query), + SketchParams::DDSketch { .. } => ddsketch::guarantee(algorithm, params, query), + SketchParams::Hll { .. } => hll::generic_guarantee(algorithm, params, query), + SketchParams::Cms { .. } | SketchParams::CmsWithHeap { .. } => { + cms::guarantee(algorithm, params, query) + } + SketchParams::CountSketch { .. } | SketchParams::CountSketchWithHeap { .. } => { + count_sketch::guarantee(algorithm, params, query) + } + SketchParams::Kmv { .. } => cardinality::guarantee(algorithm, params, query), + SketchParams::UnivMon { .. } => univmon::guarantee(algorithm, params, query), + SketchParams::Hydra(hydra) => { + sketch_guarantee(algorithm, &hydra.per_subpopulation(), query) + } + } +} + +fn bounded_guarantee( + algorithm: &SketchAlgorithm, + params: &SketchParams, + query: &SketchStatistic, + metric: AccuracyMetric, + bound: f64, + delta: ProbabilityExpr, + contract: &str, +) -> AccuracyGuarantee { + AccuracyGuarantee::new( + metric, + BoundExpr::Constant { value: bound }, + delta, + vec![GuaranteeSource::SketchEvaluation { + algorithm: format!("{algorithm:?}"), + contract: contract.into(), + params: serde_json::to_value(params).unwrap_or(serde_json::Value::Null), + query: format!("{query:?}"), + }], + ) +} + +/// The theoretical error bound of reading `statistic` from a state of +/// `summary_type` with parameters `parameters` at failure probability +/// `failure_probability` (the guarantee's own failure probability): the +/// algorithm's error formula (concepts doc, section 2, "Where the guarantee +/// comes from"). `None` when this θ cannot fail that rarely, when the +/// summary type does not answer `statistic` (its capability row) or has no +/// error model (`SummarySample`, `Wavelet`, `StatModel`, and sketch +/// readouts without a certified bound). +pub fn theoretical_guarantee( + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, + failure_probability: f64, +) -> Option { + match summary_type { + FieldDataType::Sketch(_) if SummaryTypeCapability::of(summary_type).answers(statistic) => { + let _ = (parameters, failure_probability); + todo!("#1207 PR 4: the algorithm's error formula at θ, at failure_probability") + } + _ => None, + } +} + +#[allow(dead_code)] // Read again for the predefined θ lists (`parameter_list`), in #1207 PR 1. +pub(crate) fn size_params( + kind: SketchAlgorithm, + intent: &AggIntent, + eps: f64, + delta: f64, +) -> SketchParams { + match kind { + SketchAlgorithm::UnivMon => univmon::size_params(intent, eps, delta), + SketchAlgorithm::Kll => kll::size_params(eps, delta), + SketchAlgorithm::Cms => SketchParams::Cms { + width: cms::cms_width(eps), + depth: cms::cms_depth(delta), + }, + SketchAlgorithm::Hll => SketchParams::Hll { + precision: hll::hll_precision(eps, delta), + }, + SketchAlgorithm::CmsWithHeap => { + let k = match intent { + AggIntent::TopK { k, .. } => *k, + _ => unreachable!("CmsWithHeap is only a TopK candidate"), + }; + SketchParams::CmsWithHeap { + width: cms::cms_width(eps), + depth: cms::cms_depth(delta), + heap_size: topk_capacity(k, eps), + } + } + // Non-preferred candidates (DDSketch / Kmv / CountSketch / + // CountSketchWithHeap) are only reachable once a cost model picks + // them; sized here so that wiring is local. + SketchAlgorithm::DDSketch => ddsketch::size_params(eps), + SketchAlgorithm::Kmv => SketchParams::Kmv { + k: cardinality::kmv_k(eps, delta), + }, + SketchAlgorithm::CountSketch => { + let width = count_sketch::count_sketch_width(eps); + SketchParams::CountSketch { + width, + depth: count_sketch::count_sketch_depth(width, delta), + } + } + SketchAlgorithm::CountSketchWithHeap => { + let k = match intent { + AggIntent::TopK { k, .. } => *k, + _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), + }; + let width = count_sketch::count_sketch_width(eps); + SketchParams::CountSketchWithHeap { + width, + depth: count_sketch::count_sketch_depth(width, delta), + heap_size: topk_capacity(k, eps), + } + } + // Listed, not `_`: a family `asap_sketchlib` adds fails to compile + // here until the planner sizes it or lists it as not offered. + SketchAlgorithm::CountSketchWithL2Norm + | SketchAlgorithm::CmsWithSpaceSaving + | SketchAlgorithm::CountSketchWithSpaceSaving + | SketchAlgorithm::CmsFold + | SketchAlgorithm::CountSketchFold + | SketchAlgorithm::HllHip + | SketchAlgorithm::KllDynamic + | SketchAlgorithm::SpaceSaving + | SketchAlgorithm::Elastic + | SketchAlgorithm::Coco + | SketchAlgorithm::Bloom + | SketchAlgorithm::SetAggregator + | SketchAlgorithm::UnivMonTwoTier + | SketchAlgorithm::UnivMonQ + | SketchAlgorithm::HydraCms + | SketchAlgorithm::HydraCountSketch + | SketchAlgorithm::HydraKll + | SketchAlgorithm::HydraHll + | SketchAlgorithm::HydraUnivMon => { + unreachable!("Pass 1 offers only the families in FAMILY_PREFERENCE, not {kind:?}") + } + } +} + +/// `P(Binomial(n, p) ≥ k)`, summed term by term in log space and rounded +/// up by a relative 1e-12 so rounding never certifies a smaller tail. +pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { + if k == 0 || p >= 1.0 { + return 1.0; + } + if k > n || p <= 0.0 { + return 0.0; + } + let (ln_p, ln_q) = (p.ln(), (-p).ln_1p()); + let mut ln_choose = 0.0; + let mut tail = 0.0; + for i in 0..=n { + if i > 0 { + ln_choose += f64::from(n - i + 1).ln() - f64::from(i).ln(); + } + if i >= k { + tail += (ln_choose + f64::from(i) * ln_p + f64::from(n - i) * ln_q).exp(); + } + } + (tail * (1.0 + 1e-12)).min(1.0) +} + +/// Accuracy-dependent candidate budget, not a completeness theorem. The +/// membership model must still certify the selected set independently. +pub(crate) fn topk_capacity(k: usize, eps: f64) -> u32 { + u32::try_from(k) + .unwrap_or(u32::MAX) + .max(saturating_ceil(1.0 / eps, 1, 1 << 26)) +} + +/// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive x saturate to `hi` +/// (a degenerate ε means "as accurate as this family goes"). +pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { + if !x.is_finite() || x <= 0.0 { + return hi; + } + (x.ceil() as u32).clamp(lo, hi) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Classic HLL's precision grows until its failure probability over the + /// source's distinct bound meets the target, up to 18. + #[test] + fn classic_hll_precision_grows_for_its_failure_probability() { + let p = hll::min_precision(0.05, 0.01, 64, 4); + assert!(p > 4); + assert!(hll::classic_failure_probability(p, 0.05, 64) <= 0.01); + assert!(hll::classic_failure_probability(p - 1, 0.05, 64) > 0.01); + assert_eq!(hll::min_precision(0.05, 1e-30, 4096, 12), 12); + } +} diff --git a/crates/accuracy/src/estimators/univmon.rs b/crates/accuracy/src/guarantee/univmon.rs similarity index 80% rename from crates/accuracy/src/estimators/univmon.rs rename to crates/accuracy/src/guarantee/univmon.rs index c9378c1d0..2bf1d8a56 100644 --- a/crates/accuracy/src/estimators/univmon.rs +++ b/crates/accuracy/src/guarantee/univmon.rs @@ -55,7 +55,6 @@ const BASELINE: SketchParams = SketchParams::UnivMon { sketch_rows: 5, sketch_cols: 1024, layers: 16, - distinct_failure_probability: None, }; pub(super) fn guarantee( @@ -213,6 +212,7 @@ fn distinct_heap(eps: f64, delta: f64) -> u32 { /// The statistic a UnivMon distinct count over `source` leaves its /// top-layer failure probability as. +#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. pub(crate) fn incomplete_top_layer_statistic(heap_size: u32, layers: u8, source: &str) -> String { format!("{INCOMPLETE_TOP_LAYER}[k={heap_size},layers={layers}]({MAX_DISTINCT}[{source}])") } @@ -251,26 +251,30 @@ pub fn incomplete_top_layer_probability(heap_size: u32, layers: u8, max_distinct } } -/// A UnivMon distinct count over `source`'s own rows: relative error at -/// most `√(1.35/(k δ_c))`, where `δ_c` is the share of the sized failure -/// probability left after the top layer's, failing with probability -/// `δ_c` plus the chance of an incomplete top layer, left symbolic in the -/// source's distinct bound. `None` for a shape that was not sized for a -/// distinct count, a heap under 8, or a shape the library would not build. -pub(crate) fn distinct_guarantee(params: &SketchParams, source: &str) -> Option { +/// A UnivMon distinct count over `source`'s own rows, at failure +/// probability `delta`: relative error at most `√(1.35/(k δ_c))`, where +/// `δ_c` is the share of `delta` left after the top layer's, failing with +/// probability `δ_c` plus the chance of an incomplete top layer, left +/// symbolic in the source's distinct bound. `None` for a heap under 8, a +/// `delta` too small for the heap, or a shape the library would not build. +#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. +pub(crate) fn distinct_guarantee( + params: &SketchParams, + delta: f64, + source: &str, +) -> Option { let SketchParams::UnivMon { heap_size, sketch_rows, sketch_cols, layers, - distinct_failure_probability: Some(delta), } = params else { return None; }; if *heap_size < DISTINCT_MIN_HEAP || !hash_independent(*sketch_rows, *sketch_cols, *layers) - || !(*delta > 0.0 && *delta < 1.0) + || !(delta > 0.0 && delta < 1.0) { return None; } @@ -306,35 +310,6 @@ pub(crate) fn distinct_guarantee(params: &SketchParams, source: &str) -> Option< )) } -/// A distinct-count `params` resized to fail with probability at most -/// `delta` at no larger relative error: the heap for its current error at -/// `delta` ([`distinct_heap`]), never smaller. Unchanged for a shape not -/// sized for a distinct count, or one that already fails no more often. -pub(crate) fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchParams { - let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - distinct_failure_probability: Some(current), - } = params - else { - return params.clone(); - }; - let chebyshev = current - incomplete_top_layer_bound(*heap_size); - if delta >= *current || chebyshev <= 0.0 { - return params.clone(); - } - let relative_error = (DISTINCT_VARIANCE / (f64::from(*heap_size) * chebyshev)).sqrt(); - SketchParams::UnivMon { - heap_size: distinct_heap(relative_error, delta).max(*heap_size), - sketch_rows: *sketch_rows, - sketch_cols: *sketch_cols, - layers: *layers, - distinct_failure_probability: Some(delta), - } -} - /// Size a UnivMon for one readout's `(eps, delta)`: /// - L2: the smallest rows and columns meeting the target ([`l2_shape`]), /// one layer (L2 reads only layer 0) and a one-key heap; @@ -354,7 +329,6 @@ pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchPar sketch_rows, sketch_cols, layers: 1, - distinct_failure_probability: None, } } AggIntent::Cardinality { .. } => SketchParams::UnivMon { @@ -362,76 +336,26 @@ pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchPar sketch_rows: 1, sketch_cols: 1, layers: MAX_LAYERS, - distinct_failure_probability: Some(delta), }, _ => BASELINE, } } -/// The heap of one UnivMon shared by two readers with heaps `heap_size` -/// and `heap_size2`, sized for a distinct count at the failure -/// probabilities they record (`None`: not sized for one): each distinct -/// reader's heap grown to the smaller failure probability at no larger -/// relative error ([`with_failure_probability`]), then the larger. The -/// shared build states its distinct guarantee at that smaller δ, where a -/// heap sized for a larger δ has a larger error, so the larger heap alone -/// could miss its reader's ε. A reader not sized for a distinct count keeps -/// its heap. -pub(crate) fn shared_heap( - heap_size: u32, - distinct_failure_probability: Option, - heap_size2: u32, - distinct_failure_probability2: Option, -) -> u32 { - let delta = match (distinct_failure_probability, distinct_failure_probability2) { - (Some(a), Some(b)) => a.min(b), - (a, b) => match a.or(b) { - Some(delta) => delta, - None => return heap_size.max(heap_size2), - }, - }; - let resized = |heap_size, distinct_failure_probability| match with_failure_probability( - &SketchParams::UnivMon { - heap_size, - sketch_rows: 1, - sketch_cols: 1, - layers: 1, - distinct_failure_probability, - }, - delta, - ) { - SketchParams::UnivMon { heap_size, .. } => heap_size, - _ => heap_size, - }; - resized(heap_size, distinct_failure_probability) - .max(resized(heap_size2, distinct_failure_probability2)) -} - #[cfg(test)] mod tests { use super::*; use asap_types::ir::operator::agg_intent::default_cardinality; fn l2_intent() -> AggIntent { - AggIntent::FrequencyL2 { - col: None, - accuracy: AccuracyTarget::exact(), - } + AggIntent::FrequencyL2 { col: None } } - fn univmon( - heap_size: u32, - rows: u32, - cols: u32, - layers: u8, - delta: Option, - ) -> SketchParams { + fn univmon(heap_size: u32, rows: u32, cols: u32, layers: u8) -> SketchParams { SketchParams::UnivMon { heap_size, sketch_rows: rows, sketch_cols: cols, layers, - distinct_failure_probability: delta, } } @@ -443,16 +367,6 @@ mod tests { ) } - fn meets(guarantee: &AccuracyGuarantee, epsilon: f64, delta: f64) -> bool { - satisfies( - guarantee, - &crate::native_target( - &AccuracyTarget::max_error(epsilon).failure_probability(delta), - guarantee, - ), - ) - } - fn with(params: &SketchParams, f: impl FnOnce(&mut u32, &mut u32, &mut u32)) -> SketchParams { let mut params = params.clone(); let SketchParams::UnivMon { @@ -486,17 +400,17 @@ mod tests { let guarantee = l2(¶ms).expect("an L2 guarantee"); assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); assert!( - meets(&guarantee, eps, delta), + meets(&guarantee, eps, Some(delta)), "({eps}, {delta}): {params:?}" ); let fewer_rows = with(¶ms, |_, rows, _| *rows -= 1); assert!( - l2(&fewer_rows).is_none_or(|g| !meets(&g, eps, delta)), + l2(&fewer_rows).is_none_or(|g| !meets(&g, eps, Some(delta))), "({eps}, {delta}): {fewer_rows:?}" ); let fewer_cols = with(¶ms, |_, _, cols| *cols -= 1); assert!( - l2(&fewer_cols).is_none_or(|g| !meets(&g, eps, delta)), + l2(&fewer_cols).is_none_or(|g| !meets(&g, eps, Some(delta))), "({eps}, {delta}): {fewer_cols:?}" ); } @@ -529,9 +443,9 @@ mod tests { #[test] fn l2_shapes_stay_within_the_hash_budget() { // 8 · (16 + 1) = 136 > 128. - assert!(l2(&univmon(1, 8, 1 << 16, 1, None)).is_none()); - assert!(l2(&univmon(1, 7, 1 << 16, 1, None)).is_some()); - assert!(l2(&univmon(1, 5, 1024, 20, None)).is_none()); + assert!(l2(&univmon(1, 8, 1 << 16, 1)).is_none()); + assert!(l2(&univmon(1, 7, 1 << 16, 1)).is_some()); + assert!(l2(&univmon(1, 5, 1024, 20)).is_none()); for (eps, delta) in [(0.001, 1e-6), (1e-5, 0.01), (0.01, 1e-12)] { let params = size_params(&l2_intent(), eps, delta); let SketchParams::UnivMon { @@ -547,7 +461,10 @@ mod tests { hash_independent(sketch_rows, sketch_cols, layers), "{params:?}" ); - assert!(!meets(&l2(¶ms).unwrap(), eps, delta), "{params:?}"); + assert!( + !meets(&l2(¶ms).unwrap(), eps, Some(delta)), + "{params:?}" + ); } } @@ -558,8 +475,8 @@ mod tests { fn univmon_past_twenty_rows_is_not_hash_independent() { assert!(hash_independent(20, 32, 1)); assert!(!hash_independent(21, 32, 1)); - assert!(l2(&univmon(1, 20, 32, 1, None)).is_some()); - assert!(l2(&univmon(1, 21, 32, 1, None)).is_none()); + assert!(l2(&univmon(1, 20, 32, 1)).is_some()); + assert!(l2(&univmon(1, 21, 32, 1)).is_none()); } /// Each distinct-count heap meets its target once the source's bound @@ -569,7 +486,8 @@ mod tests { for (eps, delta) in TARGETS { let params = size_params(&default_cardinality(), eps, delta); let bind = |params: &SketchParams, max_distinct| { - let guarantee = distinct_guarantee(params, "m").expect("a distinct guarantee"); + let guarantee = + distinct_guarantee(params, delta, "m").expect("a distinct guarantee"); guarantee.bind(&|statistic| { let (k, layers, source) = parse_incomplete_top_layer_statistic(statistic)?; (source == "m") @@ -580,12 +498,12 @@ mod tests { let guarantee = bind(¶ms, max_distinct); assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); assert!( - meets(&guarantee, eps, delta), + meets(&guarantee, eps, Some(delta)), "({eps}, {delta}): {params:?}" ); let smaller = with(¶ms, |heap, _, _| *heap -= 1); assert!( - !meets(&bind(&smaller, max_distinct), eps, delta), + !meets(&bind(&smaller, max_distinct), eps, Some(delta)), "({eps}, {delta}): {smaller:?}" ); } @@ -608,9 +526,9 @@ mod tests { #[test] fn distinct_guarantee_needs_the_source_bound_and_enough_layers() { let params = size_params(&default_cardinality(), 0.05, 0.01); - let guarantee = distinct_guarantee(¶ms, "m").unwrap(); + let guarantee = distinct_guarantee(¶ms, 0.01, "m").unwrap(); assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - assert!(!meets(&guarantee, 0.05, 0.01)); + assert!(!meets(&guarantee, 0.05, Some(0.01))); let SketchParams::UnivMon { heap_size: k, .. } = params else { unreachable!() }; @@ -628,15 +546,13 @@ mod tests { assert_eq!(incomplete_top_layer_probability(8, 19, u64::MAX), 1.0); } - /// Only a shape sized for a distinct count, with a heap of at least 8 - /// and a buildable shape, gets a distinct guarantee; the guarantee - /// without a source is none. + /// Only a heap of at least 8 and a buildable shape gets a distinct + /// guarantee; the guarantee without a source is none. #[test] - fn distinct_guarantee_needs_a_distinct_sizing() { - assert!(distinct_guarantee(&univmon(256, 5, 1024, 16, None), "m").is_none()); - assert!(distinct_guarantee(&univmon(7, 1, 1, 19, Some(0.5)), "m").is_none()); - assert!(distinct_guarantee(&univmon(8, 1, 1, 20, Some(0.5)), "m").is_none()); - assert!(distinct_guarantee(&univmon(8, 1, 1, 19, Some(0.5)), "m").is_some()); + fn distinct_guarantee_needs_a_heap_of_eight_and_a_buildable_shape() { + assert!(distinct_guarantee(&univmon(7, 1, 1, 19), 0.5, "m").is_none()); + assert!(distinct_guarantee(&univmon(8, 1, 1, 20), 0.5, "m").is_none()); + assert!(distinct_guarantee(&univmon(8, 1, 1, 19), 0.5, "m").is_some()); let sized = size_params(&default_cardinality(), 0.05, 0.01); assert!(guarantee( &SketchAlgorithm::UnivMon, diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 92b623583..32ee13bd5 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -1,16 +1,18 @@ -//! `asap-accuracy` — the accuracy model the planning stages share (Pass 1, -//! Pass 2 and Stage 3): the planner's built-in analytical accuracy rules -//! (estimators, composition, guarantee propagation through a DAG, sizing, -//! budget split and target check) and the deployment's accuracy model and -//! evidence traits. +//! `asap-accuracy` — the accuracy rules the planning stages share: the +//! planner's built-in analytical rules (estimators, composition, guarantee +//! propagation through a DAG, sizing, budget split and a target's comparison +//! limit [`target_limit`]), the deployment's measured accuracy ([`AccuracyEvidence`]) and $\alpha^{\log}$ +//! of a logical edge ([`producer_satisfies_logical_requirements`]). //! -//! Estimator models derive local guarantees, composition propagates them, -//! `propagation` composes them through every node of a DAG, `sizing` sizes -//! a summary for its target, the budget split sizes each approximate input -//! of a composition for an equal share of its target, and `evidence` holds -//! the scoped contracts and statistics the rules consult. The deployment's accuracy model -//! ([`AccuracyModel`]) and evidence provider ([`AccuracyEvidenceProvider`]) -//! are read only by plan selection (Stage 3). +//! Estimator models derive theoretical guarantees, composition propagates +//! them, `propagation` composes them through every node of a DAG, `sizing` +//! sizes a summary for its target, the budget split sizes each approximate +//! input of a composition for an equal share of its target, and +//! `propagation_stats` holds the scoped contracts and statistics the rules +//! consult. `guarantee` derives each summary type's theoretical guarantee, +//! `evidence` reads the deployment's measured accuracy, `requirements` +//! decides $\alpha^{\log}$ and `error_propagation` bounds a consumer's +//! output error by its inputs'. //! See `docs/design_docs/concepts/accuracy-models.md` for the design. //! //! It depends only on `asap-types`, never on a stage crate, so every stage @@ -18,313 +20,20 @@ mod budget; pub mod composition; -pub mod estimators; +mod error_propagation; mod evidence; mod function_rules; -mod model; +pub mod guarantee; pub mod propagation; +mod propagation_stats; +mod requirements; pub mod sizing; #[cfg(feature = "test-support")] pub mod test_support; pub use budget::per_input_target; -pub use evidence::OperandDomain; -pub(crate) use evidence::PropagationStats; -pub use model::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, IgnoreAccuracyGuarantees, - NoAccuracyEvidence, -}; - -#[cfg(test)] -use asap_types::ir::properties::AccuracyScope; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::properties::{ - AccuracyError, AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, - GuaranteeSource, ProbabilityExpr, -}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; - -/// Small relative tolerance for comparing an evaluated bound against a -/// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected -/// by floating-point noise. -const SATISFACTION_TOLERANCE: f64 = 1e-9; - -/// The built-in conservative check of whether `guarantee` meets `target`: -/// a bound within its ε, and within its δ when it states one, over the -/// values the target's scope covers (a per-value δ does not meet a -/// whole-result target). When the target sets a metric, the bound must be -/// in that metric (or one with a sound direct comparison): a bound in -/// another metric does not satisfy, however small. When it sets none, a -/// bound in any metric does, each compared in its own metric. Unknown -/// required dimensions fail. Plan selection's default accuracy model -/// ([`BuiltinAccuracyModel`]) delegates to it. -pub(crate) fn satisfies(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - let within = |value: Option, limit: f64| { - value.is_some_and(|v| v <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON) - }; - if target.is_exact() || guarantee.is_exact() { - return guarantee.is_exact(); - } - target - .accuracy_metric - .is_none_or(|metric| metric_meets(guarantee.spec.accuracy_metric, metric)) - && scope_meets(guarantee, target) - && within(guarantee.spec.error.evaluate(), target.max_error) - && target - .failure_probability - .is_none_or(|delta| within(guarantee.spec.failure_probability.evaluate(), delta)) -} - -/// Whether an error bound stated in `guarantee` bounds the error `target` -/// states with the same ε, so the two numbers compare directly. The same -/// metric does; so does an L2-scaled bound for an L1-scaled target, since -/// both are scaled by the norm of the one stream the root reads and -/// `‖f‖₂ ≤ ‖f‖₁`. Any other pair needs a conversion backed by evidence -/// (Absolute ↔ Relative needs a bound on the true value's magnitude; a rank -/// error has no distribution-free value error), which the default model -/// does not make: it does not satisfy. The scope is compared separately -/// ([`scope_meets`]). -pub(crate) fn metric_meets(guarantee: AccuracyMetric, target: AccuracyMetric) -> bool { - guarantee == target - || (guarantee == AccuracyMetric::ABSOLUTE_STREAM_L2 - && target == AccuracyMetric::ABSOLUTE_STREAM_L1) -} - -/// Whether `guarantee`'s failure probability covers the values `target`'s -/// does ([`AccuracyScope::covers`](asap_types::ir::properties::AccuracyScope::covers)). A whole-result guarantee covers every -/// target, and one over the groups holding at least a share of the records -/// covers a target over a larger share (not a per-value one, whose value may -/// be a smaller group). A per-value one covers only a per-value target: it must first be stated over the -/// target's scope ([`AccuracyGuarantee::over_scope`]). The scope bounds -/// only the failure probability, so it does not matter for a target -/// without a δ requirement or for a guarantee that never fails. -fn scope_meets(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - guarantee.spec.accuracy_scope.covers(target.accuracy_scope) - || target.failure_probability.is_none() - || guarantee.spec.failure_probability.is_zero() -} - -/// `target` restated in `guarantee`'s metric, per value: the check of one -/// summary against the target it was sized for. A summary is sized in its -/// own native terms (#690 §4) and for each value it answers; an aggregate -/// intent's metric and scope are not checked: only a query root's are, by -/// plan selection. -#[cfg(test)] -pub(crate) fn native_target( - target: &AccuracyTarget, - guarantee: &AccuracyGuarantee, -) -> AccuracyTarget { - target - .clone() - .accuracy_metric(guarantee.spec.accuracy_metric) - .accuracy_scope(AccuracyScope::PerValue) -} - -#[cfg(test)] -mod tests { - use super::*; - fn abs(bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - #[test] - fn satisfies_is_fail_closed_on_unknowns_and_exact() { - let mut unknown = abs(0.0, 0.0); - unknown.spec.error = BoundExpr::Unknown { - statistic: "x".into(), - }; - assert!(!satisfies(&unknown, &AccuracyTarget::max_error(1.0))); - assert!(!satisfies(&abs(0.0, 0.01), &AccuracyTarget::exact())); - assert!(satisfies( - &AccuracyGuarantee::exact("x"), - &AccuracyTarget::exact() - )); - } - - fn guarantee(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - metric, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - - /// A bound in another metric does not satisfy a target, however small - /// its number: e.g. a Count-Min L1-scaled frequency bound never meets a - /// distinct count's relative target, and a rank bound never meets a - /// value target. An L2-scaled bound meets an L1-scaled target of the - /// same stream (`‖f‖₂ ≤ ‖f‖₁`), not the reverse; and an exact value - /// meets every metric. - #[test] - fn satisfies_requires_the_targets_metric() { - let target = |metric| { - AccuracyTarget::max_error(0.5) - .failure_probability(0.5) - .accuracy_metric(metric) - .accuracy_scope(AccuracyScope::PerValue) - }; - let (l1, l2) = ( - AccuracyMetric::ABSOLUTE_STREAM_L1, - AccuracyMetric::ABSOLUTE_STREAM_L2, - ); - for (bound, wanted, meets) in [ - (l1, l1, true), - (l2, l2, true), - (l2, l1, true), - (l1, l2, false), - (l1, AccuracyMetric::Relative, false), - (l1, AccuracyMetric::ABSOLUTE, false), - (AccuracyMetric::ABSOLUTE, l1, false), - (AccuracyMetric::Relative, AccuracyMetric::ABSOLUTE, false), - (AccuracyMetric::ABSOLUTE, AccuracyMetric::Relative, false), - (AccuracyMetric::Rank, AccuracyMetric::Relative, false), - (AccuracyMetric::Relative, AccuracyMetric::Rank, false), - (AccuracyMetric::Rank, AccuracyMetric::Rank, true), - ] { - assert_eq!( - satisfies(&guarantee(bound, 0.01, 0.01), &target(wanted)), - meets, - "{bound:?} against {wanted:?}" - ); - } - assert!(satisfies( - &AccuracyGuarantee::exact("x"), - &target(AccuracyMetric::Rank) - )); - } - - /// A target without a δ requirement checks only the error, even when - /// the guarantee's failure probability is unknown. - #[test] - fn a_target_without_delta_ignores_the_failure_probability() { - let mut g = guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.0); - g.spec.failure_probability = ProbabilityExpr::Unknown { - statistic: "x".into(), - }; - assert!(satisfies(&g, &AccuracyTarget::max_error(0.01))); - assert!(!satisfies( - &g, - &AccuracyTarget::max_error(0.01).failure_probability(0.5) - )); - } - - /// A per-value guarantee does not meet a whole-result δ target - /// directly: over `G` output values it holds for all of them with - /// δ_whole = G · δ_per ([`AccuracyGuarantee::over_scope`]), and an unbound `G` - /// leaves it unmet. A whole-result guarantee meets a per-value target - /// directly. - #[test] - fn per_value_guarantees_meet_whole_result_targets_over_g_values() { - let g = guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001); - assert_eq!(g.spec.accuracy_scope, AccuracyScope::PerValue); - let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); - assert_eq!(whole.accuracy_scope, AccuracyScope::WholeResult); - assert!(!satisfies(&g, &whole)); - let per_value = whole.clone().accuracy_scope(AccuracyScope::PerValue); - assert!(satisfies(&g, &per_value)); - let over = |values| g.over_scope(AccuracyScope::WholeResult, values); - assert_eq!( - over(Some(10)).spec.accuracy_scope, - AccuracyScope::WholeResult - ); - assert_eq!( - over(Some(10)).spec.failure_probability.evaluate(), - Some(0.01) - ); - assert!(satisfies(&over(Some(10)), &whole)); - assert!(!satisfies(&over(Some(11)), &whole)); - assert!(!satisfies(&over(None), &whole)); - assert!(satisfies(&over(Some(10)), &per_value)); - } - - /// A per-value guarantee does not meet a target on the groups holding at - /// least 5% of the records directly; stated over that scope (at most 20 - /// groups) it does when 20 · δ_per is within the target's δ. A guarantee - /// over a smaller share (more groups) meets it; one over a larger share - /// does not. - #[test] - fn a_record_share_target_is_met_over_one_over_share_groups() { - let share = AccuracyScope::GroupsWithRecordShareAtLeast; - let g = guarantee(AccuracyMetric::Relative, 0.05, 0.05 / 20.0); - let five_percent = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_scope(share(0.05)); - assert!(!satisfies(&g, &five_percent)); - assert!(satisfies(&g.over_scope(share(0.05), None), &five_percent)); - assert!(!satisfies(&g.over_scope(share(0.04), None), &five_percent)); - let one_percent_g = guarantee(AccuracyMetric::Relative, 0.05, 0.05 / 100.0); - assert!(satisfies( - &one_percent_g.over_scope(share(0.01), None), - &five_percent - )); - let over_five = g.over_scope(share(0.05), None); - assert!(!satisfies( - &over_five, - &five_percent.clone().accuracy_scope(share(0.01)) - )); - assert!(!satisfies( - &over_five, - &five_percent.clone().accuracy_scope(AccuracyScope::PerValue) - )); - } - - /// The scope bounds only the failure probability: a target without a δ - /// requirement, or a guarantee that never fails, meets a whole-result - /// target whatever the number of values. - #[test] - fn scope_matters_only_with_a_failure_probability() { - let g = guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001); - assert!(satisfies(&g, &AccuracyTarget::max_error(0.01))); - let never_fails = AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { value: 0.01 }, - ProbabilityExpr::Zero, - vec![], - ); - let whole = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Relative); - assert!(satisfies(&never_fails, &whole)); - assert!(satisfies( - &never_fails.over_scope(AccuracyScope::WholeResult, None), - &whole - )); - } - - /// A target whose metric the user did not set accepts a guarantee in - /// any metric whose own ε, δ and scope meet the target's numbers: a - /// rank, a relative or an absolute bound at any scale. A larger bound - /// or failure probability still misses it. - #[test] - fn a_target_without_a_metric_accepts_any_metric() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_scope(AccuracyScope::PerValue); - assert_eq!(target.accuracy_metric, None); - for metric in [ - AccuracyMetric::ABSOLUTE, - AccuracyMetric::ABSOLUTE_STREAM_L1, - AccuracyMetric::ABSOLUTE_STREAM_L2, - AccuracyMetric::Relative, - AccuracyMetric::Rank, - ] { - assert!( - satisfies(&guarantee(metric, 0.01, 0.01), &target), - "{metric:?}" - ); - assert!( - !satisfies(&guarantee(metric, 0.02, 0.01), &target), - "{metric:?}" - ); - assert!( - !satisfies(&guarantee(metric, 0.01, 0.02), &target), - "{metric:?}" - ); - } - } -} +pub use error_propagation::{error_propagation, ErrorPropagation}; +pub use evidence::{AccuracyEvidence, NoMeasuredGuarantees}; +pub use guarantee::theoretical_guarantee; +pub(crate) use propagation_stats::PropagationStats; +pub use requirements::{producer_satisfies_logical_requirements, target_limit}; diff --git a/crates/accuracy/src/model.rs b/crates/accuracy/src/model.rs deleted file mode 100644 index 865694196..000000000 --- a/crates/accuracy/src/model.rs +++ /dev/null @@ -1,177 +0,0 @@ -//! The deployment's accuracy model, read only by plan selection (#509 -//! Stage 3). -//! -//! Stage 1 composes each candidate's guarantee with the planner's built-in -//! analytical rules (this crate's [`composition`](crate::composition)) and -//! keeps a candidate it has no rule for with `guarantee = None`. Stage 3 decides, -//! with the model here, whether a guarantee meets its query's target (after -//! the data workload and the deployment's evidence, -//! [`AccuracyEvidenceProvider`], bind the statistics Stage 1 left unknown), -//! and whether the deployment accepts an output directly, whatever its -//! guarantee says. - -use crate::OperandDomain; -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric, AccuracyTarget}; -use asap_types::ir::{OperatorNode, QueryRoot}; - -/// The deployment's accuracy decisions in plan selection: whether a -/// guarantee meets a query's target, and whether the deployment accepts an -/// output directly, whatever its guarantee. This crate ships -/// [`BuiltinAccuracyModel`]; a deployment with its own target check (a -/// registered cross-metric conversion, say) or its own measurements -/// implements this trait and passes it in -/// `asap_planner::PlanningModels::accuracy`. -/// Guarantees are composed with the built-in rules only -/// (`propagate`). -pub trait AccuracyModel { - /// Whether `guarantee` meets `target`: a bound within its ε, and within - /// its δ when it states one, over the values the target's scope covers - /// (a per-value δ does not meet a whole-result target). When the target - /// sets a metric, the bound must be in that metric (or one with a sound - /// direct comparison): a bound in another metric does not satisfy, - /// however small. When it sets none, a bound in any metric does, each - /// compared in its own metric. Unknown required dimensions fail; - /// selection separately excludes missing accuracy evidence. - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool; - - /// Whether the deployment accepts `output`, one query's output in a - /// candidate, for `target` directly, whatever its guarantee says: - /// missing (Stage 1 had no analytical rule), met or missed. Stage 3 asks - /// this for every query output; when it returns `true`, the output is - /// accepted, and the guarantee and evidence check only records whether - /// it would have rejected it (`AcceptedAccuracy::guarantee_check_rejection`). - /// It does not override the executability check before it: a plan that - /// reads Count-Min (plain, CmsWithHeap or Hydra Count-Min) over update - /// weights not proven non-negative is rejected - /// (`RejectionCause::CountMinWeightsNotProvenNonNegative`) whatever this - /// returns, because the executor refuses CmsWithHeap and HydraCms when it - /// builds the plan and plain Count-Min adds a negative weight, which voids - /// its never-underestimate bound. The default accepts - /// none, so Stage 3 checks the Stage 1 guarantee, then the evidence - /// ([`AccuracyEvidenceProvider::output_guarantee`]) with - /// [`Self::satisfies`]. - fn accepts_output_regardless_of_guarantee( - &self, - _output: &QueryRoot, - _target: &AccuracyTarget, - ) -> bool { - false - } - - /// Whether [`Self::accepts_output_regardless_of_guarantee`] may accept - /// some output. Plan selection discards a query's candidate before - /// composition for a missing or unknown guarantee only when this is - /// `false`: otherwise the model might accept the same plan with larger - /// summaries, which sharing with another query can give it. The default - /// is `true`, which keeps every such candidate. - fn may_accept_regardless_of_guarantee(&self) -> bool { - true - } - - /// Whether a guarantee stated in `metric` can meet `target` with some - /// error bound and failure probability, however small: Pass 1 prunes a - /// summary with which no reading query's output can. The default is - /// [`Self::satisfies`]'s built-in rule: an exact target accepts only an - /// exact output, and a target that sets a metric only a bound in that - /// metric (or an L2-scaled bound for an L1-scaled target). A model that - /// converts between metrics overrides it. - fn metric_can_meet(&self, metric: AccuracyMetric, target: &AccuracyTarget) -> bool { - !target.is_exact() - && target - .accuracy_metric - .is_none_or(|wanted| crate::metric_meets(metric, wanted)) - } -} - -/// The planner's built-in analytical rules as an [`AccuracyModel`]: the -/// conservative target check `satisfies` makes, and no -/// output accepted regardless of its guarantee. -/// The default in `asap_planner::PlanningModels::builtin`. -#[derive(Debug, Default, Clone, Copy)] -pub struct BuiltinAccuracyModel; - -impl AccuracyModel for BuiltinAccuracyModel { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - crate::satisfies(guarantee, target) - } - - fn may_accept_regardless_of_guarantee(&self) -> bool { - false - } -} - -/// The deployment's planning-time accuracy evidence, read only by plan -/// selection: enforced operand domains the propagation rules need, and -/// empirical guarantees for whole query outputs. Passed in -/// `asap_planner::PlanningModels::evidence`. -pub trait AccuracyEvidenceProvider { - /// Enforced domain of `operand`'s true values ([`OperandDomain`]), scoped - /// to this complete expression, including its source, filters, grouping - /// and window: the operands of a product or quotient of estimates whose - /// errors are not both relative. `None` means unknown. - fn operand_domain(&self, _operand: &OperatorNode) -> Option { - None - } - - /// The deployment's empirical accuracy for one candidate's whole query - /// output, as a guarantee. Plan selection (#509 Stage 3) accepts a - /// candidate whose output has no theoretical guarantee that meets the - /// target (e.g. a difference of two quantile estimates) when this shows - /// the output meets it. Stage 1 never reads it: root guarantees there - /// come from the built-in analytical rules. `None` means no evidence. - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - None - } - - /// Whether this provider may return any evidence: an operand domain or - /// an output guarantee. Plan selection discards a query's candidate - /// before composition for a missing or unknown guarantee only when this - /// is `false`: otherwise evidence might exist for the same plan with - /// larger summaries, which sharing with another query can give it. The - /// default is `true`, which keeps every such candidate. - fn may_provide_evidence(&self) -> bool { - true - } -} - -/// No deployment evidence: every operand domain and output guarantee is -/// unknown. The default in `asap_planner::PlanningModels::builtin`. -#[derive(Debug, Default, Clone, Copy)] -pub struct NoAccuracyEvidence; - -impl AccuracyEvidenceProvider for NoAccuracyEvidence { - fn may_provide_evidence(&self) -> bool { - false - } -} - -/// Accepts every query output regardless of its guarantee, for comparing -/// candidates on cost alone (e.g. a benchmark of sketch plans that have no -/// analytical guarantee); [`Self::satisfies`] is the built-in rule. The -/// facade passes it to plan selection when the caller asks to ignore -/// accuracy guarantees (`PlanningModels::ignore_accuracy_guarantees`). -/// Executability checks still run: Count-Min over update weights not proven -/// non-negative is rejected. Each output accepted only because of it is -/// marked in the report (`AcceptedAccuracy::guarantee_check_rejection`). -#[derive(Debug, Default, Clone, Copy)] -pub struct IgnoreAccuracyGuarantees; - -impl AccuracyModel for IgnoreAccuracyGuarantees { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - BuiltinAccuracyModel.satisfies(guarantee, target) - } - - fn accepts_output_regardless_of_guarantee( - &self, - _output: &QueryRoot, - _target: &AccuracyTarget, - ) -> bool { - true - } - - /// It accepts every output, so no candidate is discarded early for - /// its accuracy. - fn may_accept_regardless_of_guarantee(&self) -> bool { - true - } -} diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index ed33978df..c50268744 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -6,32 +6,31 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; -use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::tumbling_window_chain::{merged_chain, merged_window_build}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::guarantee::{ +use asap_types::accuracy::{ AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, GuaranteeSource, ProbabilityExpr, }; +use asap_types::ir::operator::arguments::{BinaryOpKind, JoinKind, Reduction, Source}; +use asap_types::ir::operator::tumbling_window_chain::merged_chain; +use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::FieldDataType; use asap_types::ir::schema::{DataType, Schema}; -use asap_types::ir::schema::{ - ExactKind, FamilyCapability, FieldDataType, GroupingStrategy, MergedError, - NonNegativeWeightProof, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, - SummaryInputExpr, SummaryUpdate, WeightDomain, -}; use asap_types::ir::{ ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, }; +use asap_types::summary_type::{ + ExactKind, MergedError, NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, + SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, WeightDomain, +}; use crate::composition::{ additive_as_absolute, propagate, raw_data_samples_sign_statistic, StreamNorm, GROUP_ROW_COUNT, PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, }; -use crate::estimators::local_guarantee; -use crate::estimators::{hll, univmon}; + use crate::function_rules::approximate_input_composition; use crate::PropagationStats; @@ -97,10 +96,10 @@ pub fn approximate_inputs(root: &QueryRoot) -> Vec> { node.asap(), Some( ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. } | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. } ) @@ -122,7 +121,7 @@ pub fn root_guarantee(root: &QueryRoot) -> Option { /// Bind known statistics in existing contracts, then recover compositions /// that previously required constant child bounds. Physical state contracts, -/// schemas, timing, materialization, and sharing are preserved. This does not +/// schemas and sharing are preserved. This does not /// infer facts: `bind` must leave unsupported statistics unknown. pub fn bind_guarantees( root: &QueryRoot, @@ -137,9 +136,7 @@ pub fn bind_guarantees( return done.clone(); } let operator = node.operator.map_children(|child| walk(child, bind, cache)); - let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization); + let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()); rebuilt.guarantee = node .guarantee .as_ref() @@ -159,90 +156,6 @@ pub fn bind_guarantees( root_guarantee(&bound) } -/// `roots` with every sketch summary state rebuilt with the parameters -/// `resize` gives it (same algorithm; `None` keeps them), and every node -/// above one rebuilt over its new inputs with its guarantee composed again -/// ([`composed_guarantee`]; a node that had none keeps none). Timing, -/// materialization and sharing are kept. Pass 1 sizes summaries -/// for a failure probability this way. -pub fn with_resized_summaries( - roots: &[QueryRoot], - resize: &dyn Fn(&OperatorNode, &SketchParams) -> Option, -) -> Result, SchemaDerivationError> { - fn walk( - node: &Rc, - resize: &dyn Fn(&OperatorNode, &SketchParams) -> Option, - cache: &mut NodeToRewrittenNodeCache, - ) -> Result, SchemaDerivationError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - walk(child, resize, cache)?; - } - let mut changed = node - .children() - .iter() - .any(|child| !Rc::ptr_eq(child, &cache[&Rc::as_ptr(child)])); - let mut operator = node.operator.clone(); - if let Operator::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) = &mut operator - { - if let Some(params) = resize(node, kind.params()).filter(|p| p != kind.params()) { - *kind = SketchKind::new(*kind.algorithm(), params); - changed = true; - } - } - let rebuilt = if changed { - let template = OperatorNode { - operator, - ..(**node).clone() - }; - let mut rebuilt = - template.with_new_children(|child| cache[&Rc::as_ptr(child)].clone())?; - rebuilt.guarantee = node - .guarantee - .as_ref() - .and_then(|_| composed_guarantee(&rebuilt)); - Rc::new( - rebuilt - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - node.clone() - }; - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - Ok(rebuilt) - } - let mut cache = HashMap::new(); - roots - .iter() - .map(|root| { - Ok(match root { - QueryRoot::Operator(node) => QueryRoot::Operator(walk(node, resize, &mut cache)?), - QueryRoot::Scalar(expr) => { - let mut rebuilt = HashMap::new(); - for node in expr.operator_refs() { - rebuilt.insert(Rc::as_ptr(node), walk(node, resize, &mut cache)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| rebuilt[&Rc::as_ptr(node)].clone()), - ) - } - }) - }) - .collect() -} - /// The one source a summary state counts the distinct items of its own /// rows of, as `source_distinct_guarantee` reads it: the source whose /// distinct bound gives its failure probability. `None` for a state over @@ -398,7 +311,7 @@ fn selected_group_guarantee(node: &OperatorNode) -> Option { /// failure probability per point ([`AccuracyScope::PerValue`]). Plan /// selection states it over a whole result of `G` points as `G·δ`. /// -/// [`AccuracyScope::PerValue`]: asap_types::ir::properties::AccuracyScope::PerValue +/// [`AccuracyScope::PerValue`]: asap_types::accuracy::AccuracyScope::PerValue pub fn composed_guarantee(node: &OperatorNode) -> Option { if let Some(guarantee) = selected_group_guarantee(node) { return Some(guarantee); @@ -432,13 +345,13 @@ pub fn composed_guarantee(node: &OperatorNode) -> Option { match &node.operator { Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match child.asap() { Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(..), + summary_type: FieldDataType::ExactAggregate(..), .. }) if exact_inputs(child) => Some(AccuracyGuarantee::exact( "exact accumulator over exact input", )), Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(kind, _), + summary_type: FieldDataType::ExactAggregate(kind, _), child: input, reduction, .. @@ -463,7 +376,7 @@ pub fn composed_guarantee(node: &OperatorNode) -> Option { query, }) => match summary_input.asap() { Some(ASAPOp::SummaryAgg { - family, + summary_type: family, input: update, child: input, reduction, @@ -474,7 +387,9 @@ pub fn composed_guarantee(node: &OperatorNode) -> Option { let local = match estimate_bound_over_raw_data_samples(family, update, query) { Some(gated) => gated, None => source_distinct_guarantee(family, query, input) - .or_else(|| local_guarantee(family, query))?, + .or_else(|| { + todo!("#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters") + })?, }; let local = interpolation_guarantee(family, update, query, input, local)?; if exact_inputs(summary_input) { @@ -583,12 +498,15 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> return Some(None); }; // Kept whole, a selected state keeps the guarantee it was built with. - let input = asap_types::ir::operator::asap::selected_from(input).unwrap_or(input); + let input = + asap_types::ir::operator::asap::selected_from(input).unwrap_or_else(|| Rc::clone(input)); let family = state.asap()?.produced_state()?; let merged = match node.asap()? { - ASAPOp::SummaryEstimate { query, .. } => FamilyCapability::of(family).merged_error(query), + ASAPOp::SummaryEstimate { query, .. } => { + SummaryTypeCapability::of(family).merged_error(query) + } // An exact accumulator merged across groups is exact. - _ => FamilyCapability::merges_groups(family).then_some(MergedError::Exact), + _ => SummaryTypeCapability::merges_groups(family).then_some(MergedError::Exact), }; if !matches!( merged, @@ -596,44 +514,43 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> ) { return Some(None); } - let reader = OperatorNode::with_schema( + let consumer = OperatorNode::with_schema( node.operator.map_children(|child| { if Rc::ptr_eq(child, state) { - Rc::clone(input) + Rc::clone(&input) } else { Rc::clone(child) } }), node.schema.clone(), ); - Some(composed_guarantee(&reader)) + Some(composed_guarantee(&consumer)) } /// The guarantee of an estimate or finalized value read from merged /// tumbling windows ([`merged_chain`]): the guarantee of the same read of one -/// build over the merged tumbling windows' time range ([`merged_window_build`]: the whole -/// chain's, or its newest tumbling windows') when the +/// build over the merged tumbling windows' time range (the whole chain's, or +/// its newest tumbling windows') when the /// family's merged error for that read is exact or the same as that build's /// (the family-capability table); none when it is approximate (evidence /// only, for plan selection to judge) or the family has no merge. `None` for /// any node that does not read merged tumbling windows. fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option> { - let (state, merged) = match node.asap()? { + let merged = match node.asap()? { ASAPOp::SummaryEstimate { summary_input, query, } => { - let family = merged_chain(summary_input)?.asap()?.produced_state()?; - ( - summary_input, - FamilyCapability::of(family).merged_error(query), - ) + let chain = merged_chain(summary_input)?; + let family = chain.asap()?.produced_state()?; + SummaryTypeCapability::of(family).merged_error(query) } ASAPOp::FinalizeExactAccumulator { child } => { - let family = merged_chain(child)?.asap()?.produced_state()?; + let chain = merged_chain(child)?; + let family = chain.asap()?.produced_state()?; // An exact accumulator's merged state is exact. - let merged = FamilyCapability::of(family).merge_inputs; - (child, merged.map(|_| MergedError::Exact)) + let merged = SummaryTypeCapability::of(family).merge_inputs; + merged.map(|_| MergedError::Exact) } _ => return None, }; @@ -643,33 +560,9 @@ fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option, ) -> Option { - let FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) = family else { + let FieldDataType::Sketch(algorithm) = family else { return None; }; if *query != SketchStatistic::Cardinality { return None; } source_scan(input)?; - match kind.params() { - SketchParams::Hll { - precision, - relative_error, - } => hll::classic_guarantee(*precision, *relative_error, &source(input)?), - params @ SketchParams::UnivMon { .. } => { - univmon::distinct_guarantee(params, &source(input)?) + match algorithm { + SketchAlgorithm::Hll | SketchAlgorithm::UnivMon => { + todo!( + "#1207 PR 4: the distinct-count guarantee of θ from LogicalProperties::parameters" + ) } _ => None, } @@ -763,7 +654,7 @@ pub(crate) fn estimates_raw_data_samples_of(node: &OperatorNode) -> Option match child.asap()? { ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, _), child, filter: None, .. @@ -779,7 +670,7 @@ pub(crate) fn estimates_raw_data_samples_of(node: &OperatorNode) -> Option match merged_chain(summary_input) - .unwrap_or(summary_input) + .unwrap_or_else(|| Rc::clone(summary_input)) .asap()? { ASAPOp::SummaryAgg { @@ -1219,10 +1110,10 @@ pub(crate) fn stream_norm(node: &Rc) -> Option { matches!( node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), + summary_type: FieldDataType::Sketch(algorithm), .. }) if matches!( - kind.algorithm(), + algorithm, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketch @@ -1598,8 +1489,8 @@ pub(crate) fn estimate_has_bound( ) -> bool { let count_min = matches!( family, - FieldDataType::Sketch(kind, _) - if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) + FieldDataType::Sketch(algorithm) + if matches!(algorithm, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) ); !matches!(query, SketchStatistic::TopK { .. }) && !(count_min && !matches!(update.weight_domain, WeightDomain::NonNegative { .. })) @@ -1616,8 +1507,7 @@ fn interpolation_guarantee( input: &OperatorNode, mut guarantee: AccuracyGuarantee, ) -> Option { - let interpolates = matches!(family, FieldDataType::Sketch(kind, _) - if *kind.algorithm() == SketchAlgorithm::DDSketch) + let interpolates = matches!(family, FieldDataType::Sketch(SketchAlgorithm::DDSketch)) && matches!(query, SketchStatistic::Quantile { q } if *q > 0.0 && *q < 1.0); if !interpolates || matches!(update.weight_domain, WeightDomain::NonNegative { .. }) { return Some(guarantee); @@ -1633,10 +1523,10 @@ fn interpolation_guarantee( } _ => return None, }; - guarantee.spec.error = asap_types::ir::properties::BoundExpr::Product { + guarantee.spec.error = asap_types::accuracy::BoundExpr::Product { factors: vec![ guarantee.spec.error, - asap_types::ir::properties::BoundExpr::Unknown { + asap_types::accuracy::BoundExpr::Unknown { statistic: raw_data_samples_sign_statistic(&metric), }, ], @@ -1666,16 +1556,8 @@ pub(crate) fn estimate_bound_over_raw_data_samples( if !estimate_has_bound(family, &proven, query) { return None; } - let mut local = local_guarantee(family, query)?; - local.spec.error = asap_types::ir::properties::BoundExpr::Product { - factors: vec![ - local.spec.error, - asap_types::ir::properties::BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic(metric), - }, - ], - }; - Some(local) + let _ = metric; + todo!("#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters, its bound gated on the metric's sign") } pub fn split_and(expr: &ScalarExpr, out: &mut Vec) { diff --git a/crates/accuracy/src/propagation_stats.rs b/crates/accuracy/src/propagation_stats.rs new file mode 100644 index 000000000..58db409b9 --- /dev/null +++ b/crates/accuracy/src/propagation_stats.rs @@ -0,0 +1,39 @@ +//! Scoped source contracts and statistics the accuracy rules consult. +use asap_types::accuracy::GuaranteeSource; + +/// Statistics a propagation rule may consult. Every field is optional and +/// defaults to "unknown": a rule that needs a missing statistic emits a +/// [`BoundExpr::Unknown`] leaf (or rejects) rather than guessing. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct PropagationStats { + /// Provenance for supplied evidence (source, observation identity, etc.). + pub(crate) evidence_provenance: Vec, + /// Whether every input value is known to be non-negative — required by + /// the multiplicative relative-error rule, which is unsound across a + /// sign change. + pub(crate) values_non_negative: Option, + /// The statistic an unknown sign is named by when the folded values are + /// one metric's raw data samples or plain sums of them + /// ([`raw_data_samples_sign_statistic`](crate::composition::raw_data_samples_sign_statistic)): + /// the relative bound is then gated on it, and plan selection binds it + /// to 1 when the metric is declared a counter. `None`: the bare + /// `values_non_negative`, which nothing binds. + pub(crate) sign_statistic: Option, + /// Number of input rows an exact aggregation consumes (e.g. the number + /// of groups a function folds), for exact aggregate union bounds + /// bound over per-input failures. + pub(crate) input_row_count: Option, + /// The statistic an unknown row count is named by (default + /// `input_row_count`), so a fold whose rows the data workload's input + /// cardinality does not bound can leave a statistic nothing binds. + pub(crate) row_count_statistic: Option<&'static str>, + /// Lower confidence bound of the kth selected TopK item, after widening + /// the interval by the sketch's own estimation error. + pub(crate) topk_selected_lower_bound: Option, + /// Greatest upper confidence bound among excluded TopK items, after + /// widening the interval by the sketch's own estimation error. + pub(crate) topk_excluded_upper_bound: Option, + /// Union-bound failure probability of all intervals used by the margin + /// certificate. + pub(crate) topk_interval_failure_probability: Option, +} diff --git a/crates/accuracy/src/requirements.rs b/crates/accuracy/src/requirements.rs new file mode 100644 index 000000000..58e7d78d6 --- /dev/null +++ b/crates/accuracy/src/requirements.rs @@ -0,0 +1,65 @@ +//! Whether a producer meets a consumer's logical requirements: +//! $\alpha^{\log}$ of a logical edge, and the limit an evaluated bound is +//! compared against. + +#[cfg(test)] +use asap_types::accuracy::AccuracyGuarantee; +use asap_types::stages::{CandidateForMemoGroupId, Memo}; + +use crate::AccuracyEvidence; + +/// $\alpha^{\log}$ of one edge (concepts doc, section 2): for a readout's +/// input, the producer's guarantee for the readout's statistic at the +/// required failure probability is within the required error, and a +/// top-k's heap holds k; for any other input, the metric rule (section 4). +pub fn producer_satisfies_logical_requirements( + memo: &Memo, + producer: CandidateForMemoGroupId, + consumer: CandidateForMemoGroupId, + input_index_of_consumer: usize, + evidence: &dyn AccuracyEvidence, +) -> bool { + let _ = (memo, producer, consumer, input_index_of_consumer, evidence); + todo!("#1207 PR 4: the readout rule against RequiredAccuracy, else the metric rule") +} + +/// Small relative tolerance for comparing an evaluated bound against a +/// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected +/// by floating-point noise. +const SATISFACTION_TOLERANCE: f64 = 1e-9; + +/// The largest evaluated error (or failure probability) that meets a +/// required `limit`: `limit` widened by [`SATISFACTION_TOLERANCE`], so a +/// bound equal to it up to floating-point noise meets it. +pub fn target_limit(limit: f64) -> f64 { + limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON +} + +/// Whether `guarantee`'s evaluated error is within `max_error` and, when +/// `failure_probability` is set, its evaluated failure probability within +/// it ([`target_limit`]); unknown values do not meet. +#[cfg(test)] +pub(crate) fn meets( + guarantee: &AccuracyGuarantee, + max_error: f64, + failure_probability: Option, +) -> bool { + let within = |value: Option, limit: f64| value.is_some_and(|v| v <= target_limit(limit)); + within(guarantee.spec.error.evaluate(), max_error) + && failure_probability + .is_none_or(|delta| within(guarantee.spec.failure_probability.evaluate(), delta)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A bound equal to the limit up to floating-point noise meets it; a + /// bound one part in a million above it does not. + #[test] + fn target_limit_admits_only_floating_point_noise() { + assert!(0.01 * (1.0 + 1e-12) <= target_limit(0.01)); + assert!(0.01 * (1.0 + 1e-6) > target_limit(0.01)); + assert!(0.0 <= target_limit(0.0)); + } +} diff --git a/crates/accuracy/src/sizing.rs b/crates/accuracy/src/sizing.rs index 4960f3d14..25e5c2410 100644 --- a/crates/accuracy/src/sizing.rs +++ b/crates/accuracy/src/sizing.rs @@ -1,28 +1,9 @@ -//! Sizing: the target an approximate-capable intent's summary is sized -//! against, its `(ε, δ)` budget, the default sketch parameters for it, a -//! family sized for one equal share of that target, and the failure -//! probability each approximate input needs over the values the target -//! covers. -use std::rc::Rc; +//! Sizing: the statistic a summary is read for, and the default sketch +//! parameters for an `(ε, δ)`. use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, -}; -use asap_types::ir::{OperatorNode, QueryRoot, SchemaDerivationError}; -use asap_types::workload::bounds::{series_read, values_per_result}; -use asap_types::workload::DataWorkload; - -use crate::estimators::{hll, local_guarantee, with_failure_probability}; -use crate::per_input_target; -use crate::propagation::{approximate_inputs, distinct_bound_source, with_resized_summaries}; - -/// Confidence δ assumed when the target carries only an ε -/// (`AccuracyTarget::epsilon`): the (ε, δ)-parameterised sketches (CMS) need -/// one. `ln(1/0.01) → depth 5`, matching the conventional CMS sizing. -pub(crate) const DEFAULT_DELTA: f64 = 0.01; +use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; /// The statistic a summary is read for to answer `intent`, or `None` when /// no sketch statistic serves the intent. @@ -33,7 +14,7 @@ pub fn sketch_statistic(intent: &AggIntent) -> Option { AggIntent::FrequencyL2 { .. } => SketchStatistic::FrequencyL2, AggIntent::FrequencyEntropy { .. } => SketchStatistic::FrequencyEntropy, AggIntent::TopK { k, .. } => SketchStatistic::TopK { k: *k }, - AggIntent::Count { .. } => SketchStatistic::PointCount { + AggIntent::Count => SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, }, @@ -41,57 +22,6 @@ pub fn sketch_statistic(intent: &AggIntent) -> Option { }) } -/// The [`AccuracyTarget`] an approximate-capable intent's summary is sized -/// and checked against, or `None` for every other intent (no sketch -/// candidate applies): the query's target restated in the intent's own metric -/// (#690 §4): an L1-scaled absolute error for counts and top-k frequencies, -/// a relative error for distinct counts and L2 frequency moments, an -/// absolute error for entropy, and for a quantile the query's metric or a -/// rank error when none is set. The query's own metric, set or not, is -/// checked at its root in Stage 3. -pub fn sizing_target(intent: &AggIntent) -> Option { - let (target, metric) = match intent { - AggIntent::Quantile { accuracy, .. } => ( - accuracy, - accuracy.accuracy_metric.unwrap_or(AccuracyMetric::Rank), - ), - AggIntent::Cardinality { accuracy, .. } | AggIntent::FrequencyL2 { accuracy, .. } => { - (accuracy, AccuracyMetric::Relative) - } - AggIntent::FrequencyEntropy { accuracy, .. } => (accuracy, AccuracyMetric::ABSOLUTE), - AggIntent::Count { accuracy } | AggIntent::TopK { accuracy, .. } => { - (accuracy, AccuracyMetric::ABSOLUTE_STREAM_L1) - } - _ => return None, - }; - Some(target_in(target, metric)) -} - -/// `target`'s ε, δ and scope stated in `metric`. An exact target stays -/// exact with no metric: exact is exact in every metric. -fn target_in(target: &AccuracyTarget, metric: AccuracyMetric) -> AccuracyTarget { - if target.is_exact() { - return AccuracyTarget::exact().accuracy_scope(target.accuracy_scope); - } - target.clone().accuracy_metric(metric) -} - -/// Resolve an [`AccuracyTarget`] into the `(eps, delta)` budget -/// `default_size_params` needs — one place this resolution happens, so -/// nothing can drift apart on it. -/// -/// An exact target degrades to the tightest parameters for a caller that -/// resolves it anyway. -pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { - if accuracy.is_exact() { - return (f64::MIN_POSITIVE, DEFAULT_DELTA); - } - ( - accuracy.max_error, - accuracy.failure_probability.unwrap_or(DEFAULT_DELTA), - ) -} - /// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved /// `(eps, delta)` accuracy budget. /// @@ -99,195 +29,24 @@ pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { /// smallest parameter satisfying the target, clamped to the family's sane /// range. A non-positive ε saturates to the clamp maximum (tightest /// allowed). +#[allow(dead_code)] // Read by `test_support` until #1207 PR 1 replaces it with `parameter_list`. pub(crate) fn default_size_params( kind: SketchAlgorithm, intent: &AggIntent, eps: f64, delta: f64, ) -> SketchParams { - crate::estimators::size_params(kind, intent, eps, delta) -} - -/// `algorithm` sized for one of `share` equal shares of `intent`'s accuracy -/// requirement ([`per_input_target`]), split in the metric of the summary's own -/// bound: a relative error splits so products and quotients of the shares -/// meet it. -pub fn sized(algorithm: &SketchAlgorithm, intent: &AggIntent, share: usize) -> SketchKind { - let target = sizing_target(intent).unwrap_or_else(AccuracyTarget::exact); - let size = |target: &AccuracyTarget| { - let (epsilon, delta) = accuracy_budget(target); - SketchKind::new( - *algorithm, - default_size_params(*algorithm, intent, epsilon, delta), - ) - }; - let whole = size(&target); - if share <= 1 { - return whole; - } - let metric = sketch_statistic(intent) - .and_then(|query| { - local_guarantee( - &FieldDataType::Sketch(whole.clone(), GroupingStrategy::default()), - &query, - ) - }) - .map(|guarantee| guarantee.spec.accuracy_metric) - .or(target.accuracy_metric); - // The split needs a concrete metric. A summary without an error model - // whose intent's target sets none is not split (never a guessed - // metric): it stays sized for the whole target, and Stage 3 rejects the - // candidate if its composed output then misses the query's target. - let Some(metric) = metric else { - debug_assert!( - target.is_exact(), - "sizing {algorithm:?} for a share of an approximate target without a metric" - ); - return whole; - }; - per_input_target(&target, metric, share).map_or(whole, |share| size(&share)) -} - -/// The failure probability each of the `budget_parts` approximate inputs -/// of query `root`'s value is sized for, so that the value meets `target` -/// over every value the target covers: `δ / (n · V)`, the query's `δ` -/// shared equally by `n = budget_parts` inputs and union-bounded over the -/// `V` values its scope covers -/// ([`AccuracyScope::max_values_covered`](asap_types::ir::properties::AccuracyScope::max_values_covered): -/// 1 for a per-value target, at most `⌊1 / share⌋` for one over the groups -/// holding a record share, and `G` ([`values_per_result`]) for the whole -/// result). `None` for an exact target, one without a positive failure -/// probability, and a whole-result target whose `G` is unbound: its target -/// then stays unmet, and plan selection rejects the plan. -pub fn per_input_failure_probability( - target: &AccuracyTarget, - budget_parts: usize, - root: &QueryRoot, - data: &DataWorkload, -) -> Option { - let delta = target - .failure_probability - .filter(|delta| !target.is_exact() && *delta > 0.0)?; - let values = target.accuracy_scope.max_values_covered(values_per_result( - root, - series_read(root, data), - data, - ))?; - Some(delta / (budget_parts.max(1) as f64 * values as f64)) -} - -/// Sketch summary `state`'s `params` sized for failure probability -/// `failure_probability`: the parameters that depend on it enlarged -/// ([`with_failure_probability`]), and classic HLL's precision from the -/// distinct bound of the source it counts (keeping the relative error it -/// was sized for). Never smaller than `params`. `None` keeps `params`: an -/// HLL whose source declares no distinct bound. -fn sized_for_failure_probability( - state: &OperatorNode, - params: &SketchParams, - failure_probability: f64, - data: &DataWorkload, -) -> Option { - Some(match params { - SketchParams::Hll { - precision, - relative_error, - } => { - let source = distinct_bound_source(state)?; - let bound = data.sources.get(&source)?.max_distinct_per_evaluation?; - let relative_error = hll::classic_relative_error(*precision, *relative_error); - SketchParams::Hll { - precision: hll::min_precision( - relative_error, - failure_probability, - bound, - *precision, - ), - relative_error: Some(relative_error), - } - } - params => with_failure_probability(params, failure_probability), - }) -} - -/// `roots` with every approximate input of their values -/// ([`approximate_inputs`]) sized for `failure_probability` -/// (`sized_for_failure_probability`) and the guarantees above composed -/// again ([`with_resized_summaries`]). A summary whose output carries no -/// guarantee (e.g. a top-k's heap, which bounds item frequencies, not -/// membership) is no approximate input and keeps its size: its failure -/// probability bounds nothing the target checks. The same `Rc` where -/// nothing changes. -pub fn with_summaries_sized_for_failure_probability( - roots: &[QueryRoot], - failure_probability: f64, - data: &DataWorkload, -) -> Result, SchemaDerivationError> { - let inputs: std::collections::HashSet<*const OperatorNode> = roots - .iter() - .flat_map(approximate_inputs) - .map(|state| Rc::as_ptr(&state)) - .collect(); - if inputs.is_empty() { - return Ok(roots.to_vec()); - } - with_resized_summaries(roots, &|state, params| { - inputs - .contains(&std::ptr::from_ref(state)) - .then(|| sized_for_failure_probability(state, params, failure_probability, data)) - .flatten() - }) -} - -/// Whole-workload `roots` (query `i`'s root `roots[i]` with target -/// `targets[i]`), composed outside the memo, with their summaries sized as -/// Pass 1 sizes a memo candidate: each approximate input of query `i`'s -/// value, `n_i` in all ([`approximate_inputs`]), for -/// [`per_input_failure_probability`] `δ_i / (n_i · V_i)`, a summary several -/// roots read for the smallest among them, and the guarantees above -/// composed again. For plans built by hand (tests, the fuzzer's witnesses); -/// the planner sizes its candidates in Pass 1. -pub fn with_summaries_sized_for_targets( - roots: &[QueryRoot], - targets: &[AccuracyTarget], - data: &DataWorkload, -) -> Result, SchemaDerivationError> { - let mut required: std::collections::HashMap<*const OperatorNode, f64> = - std::collections::HashMap::new(); - for (root, target) in roots.iter().zip(targets) { - let inputs = approximate_inputs(root); - let Some(failure_probability) = - per_input_failure_probability(target, inputs.len(), root, data) - else { - continue; - }; - for state in &inputs { - let entry = required - .entry(Rc::as_ptr(state)) - .or_insert(failure_probability); - *entry = entry.min(failure_probability); - } - } - if required.is_empty() { - return Ok(roots.to_vec()); - } - with_resized_summaries(roots, &|state, params| { - let failure_probability = *required.get(&std::ptr::from_ref(state))?; - sized_for_failure_probability(state, params, failure_probability, data) - }) + crate::guarantee::size_params(kind, intent, eps, delta) } #[cfg(test)] mod tests { use super::*; - /// CMS width is `⌈e/ε⌉` and depth `⌈ln(1/δ)⌉`, with an ε-only target - /// taking `DEFAULT_DELTA` (depth 5). + /// CMS width is `⌈e/ε⌉` and depth `⌈ln(1/δ)⌉`. #[test] fn epsilon_delta_sizes_cms_depth() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::max_error(0.001).failure_probability(0.001), - }; + let intent = AggIntent::Count; assert_eq!( default_size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), SketchParams::Cms { @@ -295,12 +54,8 @@ mod tests { depth: 7 } // ⌈e/0.001⌉, ⌈ln 1000⌉ ); - let target = AccuracyTarget::max_error(0.001); - let (eps, delta) = accuracy_budget(&target); - assert_eq!((eps, delta), (0.001, DEFAULT_DELTA)); - let intent = AggIntent::Count { accuracy: target }; assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, eps, delta), + default_size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), SketchParams::Cms { width: 2719, depth: 5 diff --git a/crates/accuracy/src/test_support.rs b/crates/accuracy/src/test_support.rs index 9a206a27f..67de5f443 100644 --- a/crates/accuracy/src/test_support.rs +++ b/crates/accuracy/src/test_support.rs @@ -4,18 +4,16 @@ //! constant of the same name. use std::rc::Rc; -use asap_types::ir::operator::operator_properties::Reduction; +use asap_types::accuracy::AccuracyGuarantee; +use asap_types::ir::operator::arguments::Reduction; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::guarantee::AccuracyGuarantee; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{ - FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate, -}; +use asap_types::ir::schema::FieldDataType; use asap_types::ir::{OperatorNode, ScalarExpr}; +use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate}; use crate::composition::StreamNorm; -pub use crate::evidence::PropagationStats; use crate::propagation; +pub use crate::propagation_stats::PropagationStats; /// [`propagation::composed_guarantee`]. pub fn composed_guarantee(node: &OperatorNode) -> Option { @@ -72,32 +70,19 @@ pub fn source_distinct_guarantee( propagation::source_distinct_guarantee(family, query, input) } -/// `crate::satisfies`. -pub fn satisfies(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - crate::satisfies(guarantee, target) -} - -/// `crate::estimators::local_guarantee`. -pub fn local_guarantee( - family: &FieldDataType, - query: &SketchStatistic, -) -> Option { - crate::estimators::local_guarantee(family, query) -} - -/// `crate::estimators::cms::cms_width`. +/// `crate::guarantee::cms::cms_width`. pub fn cms_width(eps: f64) -> u32 { - crate::estimators::cms::cms_width(eps) + crate::guarantee::cms::cms_width(eps) } -/// `crate::estimators::cms::cms_depth`. +/// `crate::guarantee::cms::cms_depth`. pub fn cms_depth(delta: f64) -> u32 { - crate::estimators::cms::cms_depth(delta) + crate::guarantee::cms::cms_depth(delta) } -/// `crate::estimators::kll::kll_rank_error_99`. +/// `crate::guarantee::kll::kll_rank_error_99`. pub fn kll_rank_error_99(k: u32) -> f64 { - crate::estimators::kll::kll_rank_error_99(k) + crate::guarantee::kll::kll_rank_error_99(k) } /// `crate::sizing::default_size_params`. @@ -110,9 +95,6 @@ pub fn default_size_params( crate::sizing::default_size_params(kind, intent, eps, delta) } -/// `crate::sizing::DEFAULT_DELTA`. -pub const DEFAULT_DELTA: f64 = crate::sizing::DEFAULT_DELTA; - /// `crate::composition::QUOTIENT_OPERAND_DOMAINS`. pub const QUOTIENT_OPERAND_DOMAINS: &str = crate::composition::QUOTIENT_OPERAND_DOMAINS; diff --git a/crates/cost/Cargo.toml b/crates/cost/Cargo.toml index 1436c8114..8a8e35fa9 100644 --- a/crates/cost/Cargo.toml +++ b/crates/cost/Cargo.toml @@ -22,4 +22,4 @@ asap-executor = { path = "../executor" } # by the stages above this crate; `tests/held_memory.rs` builds them with # the reference pricer's `optimize_physical_asap`. asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["reference-pricer"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index d381c59bb..1acc463c8 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -2,13 +2,15 @@ //! resource a plan uses and what that usage costs. It depends only on //! `asap-types`, so any stage can price with it. //! -//! - Resource-usage estimation: [`analytical`] and `operator_statistics` -//! (the analytical per-operator resource model), `measured` (measured -//! operation costs and their lookup), [`execution_demand`] (how often -//! each node of a physical DAG is computed), [`state_bytes`] (the bytes a -//! summary or exact state holds) and `resources` (resource dimensions -//! with explicit CPU units). -//! - Cost models: [`ResourceUsage`], [`ResourceUnitCost`], +//! - Resource-usage estimation of one node ([`node`]): +//! [`node::analytical`] and `node::operator_statistics` (the analytical +//! per-operator resource model), [`node::state_bytes`] (the bytes a +//! summary or exact state holds), `measured` (measured operation costs and +//! their lookup) and `resources` (resource dimensions with explicit CPU +//! units). +//! - The whole plan ([`plan`]): [`plan::execution_demand`] (how often each +//! node of a physical DAG is computed), its cost, retention and ranking. +//! - Cost models (`model`): [`ResourceUsage`], [`ResourceUnitCost`], //! [`ResourceSpeed`], [`CostModel`] ([`ClusterLimits`], //! [`ClusterObjective`]) and the deployment's [`CostInputs`], with the //! per-node and per-plan costs ([`NodeCost`], [`PlanCost`]). @@ -18,515 +20,29 @@ //! //! Plan selection (Stage 3, `asap-plan-selection`) documents how it prices //! a candidate with these. -use std::collections::BTreeMap; - -use asap_types::ir::physical_export::PhysicalASAPNodeId; -use asap_types::ir::properties::ExecutionTiming; - -pub mod analytical; -pub mod execution_demand; mod measured; -mod operator_statistics; -mod ranking; +mod model; +pub mod node; +pub mod plan; mod resources; -pub mod state_bytes; -mod usage; -use analytical::AnalyticalCostError; pub(crate) use measured::aggregate_lookup; pub use measured::{ - lookup, CostTableProvenance, MeasuredOperationCost, MeasuredOperationCosts, NodePriceSource, + lookup, CostTableProvenance, MeasuredOperationCost, MeasuredOperationCosts, NodeCostSource, OperatorCost, OperatorKind, StateFamily, SummaryOperation, }; -pub use operator_statistics::EdgeStatistics; -pub use ranking::{cheaper, worst_latency_ms, PlanRanking}; -pub use usage::{ - family_name, kept_tumbling_window_chains, maintained_window_ms, node_map, node_usage, - plan_cost, query_reach, retained_tumbling_windows, trailing_window, - tumbling_window_retention_ms, tumbling_window_retention_ms_of_plan, tumbling_windows_of, - NodeMap, NodeUsage, ProducerOutput, +pub use model::{ + CalibrationFit, ClusterLimits, ClusterObjective, CostInputs, CostModel, ResourceSpeed, + ResourceUnitCost, COST_SOURCE, MEASURED_COST_SOURCE, MIXED_COST_SOURCE, }; - -/// Cost per second of wall time; one unit is one CPU-millisecond under -/// [`ResourceUnitCost::illustrative`]. -pub const COST_UNIT: &str = "cost_per_second"; -/// [`PlanCost::source`] when every node is priced from the formulas. -pub const COST_SOURCE: &str = "analytical-cost-v2 (illustrative statistics)"; -/// [`PlanCost::source`] when every node is priced from the measured -/// table. -pub const MEASURED_COST_SOURCE: &str = "measured operation costs (illustrative statistics)"; -/// [`PlanCost::source`] when some nodes are priced from the measured -/// table and the others from the formulas. -pub const MIXED_COST_SOURCE: &str = - "measured operation costs and analytical-cost-v2 (illustrative statistics)"; - -/// Seconds over which non-repeating queries (one-off, scheduled and -/// unknown) are spread by default: one hour. -pub(crate) const DEFAULT_NON_REPEATING_QUERY_HORIZON_SECONDS: f64 = 3_600.0; - -/// What one candidate uses, per second of wall time unless the name says -/// otherwise, without prices: its nodes' CPU, scans, kept memory and disk -/// added up, with the candidate's peak cores and per-query latency. Every [`CostModel`] reads -/// it. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct ResourceUsage { - /// CPU operations per second: ingestion-time and query-time work. - pub cpu_ops_per_second: f64, - /// Bytes scanned per second. - pub scanned_bytes_per_second: f64, - /// Memory held between computations, averaged over time: retained - /// byte-seconds per second (ingestion-time state always, query-time - /// state while its evaluation runs). - pub average_memory_bytes: f64, - /// The most memory live at once, assuming every evaluation overlaps: - /// in-memory kept state plus each computation's working memory, once - /// per consumer that recomputes it, with the `MatAtIngestion` consumers - /// of one refresh window counted once. - pub peak_memory_bytes: u64, - /// Bytes kept on disk: state materialized on `Medium::Disk`, plus the - /// raw data query-time scans read (the longest window of each source). - pub disk_bytes: u64, - /// CPU cores to provision for the peak: the ingestion work's cores plus - /// one per query evaluating at once. - pub peak_cpu_cores: u32, - /// One per query, in workload order: its latency per evaluation. - pub latency_per_query: Vec, -} - -/// The price of one unit of each resource, per second of wall time. Cost -/// model 1 reads the usage prices (CPU, scan, memory byte-seconds, disk); -/// cost model 2 the provisioned prices (memory, CPU cores) and disk. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ResourceUnitCost { - /// Price of one CPU operation. - pub cost_per_cpu_op: f64, - /// Price of scanning one byte. - pub cost_per_scanned_byte: f64, - /// Price of holding one byte of memory for one second (cost model 1). - pub cost_per_memory_byte_second: f64, - /// Price of provisioning one byte of memory for one second (cost model 2). - pub cost_per_provisioned_memory_byte_second: f64, - /// Price of provisioning one CPU core for one second (cost model 2). - pub cost_per_provisioned_cpu_core_second: f64, - /// Price of keeping one byte on disk for one second (both models). - pub cost_per_disk_byte_second: f64, -} - -impl ResourceUnitCost { - /// The prices plan selection uses until a deployment is calibrated: - /// 1 ns of CPU per operation, 0.1 ns per scanned byte, 1 GB kept for a - /// second costs 1/8 vCPU (125 CPU-ms), held or provisioned alike; one - /// provisioned core-second is 1,000 CPU-ms. Disk is not priced (0). - pub fn illustrative() -> Self { - Self { - cost_per_cpu_op: 1e-6, - cost_per_scanned_byte: 1e-7, - cost_per_memory_byte_second: 1.25e-7, - cost_per_provisioned_memory_byte_second: 1.25e-7, - cost_per_provisioned_cpu_core_second: 1_000.0, - cost_per_disk_byte_second: 0.0, - } - } - - /// AWS Fargate on-demand prices, US East (N. Virginia), Linux/x86, for - /// cost model 1 ([`CostModel::UsageOverTime`]); one cost unit is one - /// US dollar. Source: , checked - /// 2026-10-18 (Pricing Example 1 there: $0.000011244 per vCPU-second, - /// $0.000001235 per GB-second of memory, $0.0000000308 per GB-second of - /// ephemeral storage beyond the 20 GB included). Prices change: re-check - /// the page before relying on them. - /// - /// `speed` is the deployment's [`CostInputs::resource_speed`], - /// the speed Stage 3 uses for latency, so the CPU prices charge the core - /// time latency counts. At `ResourceSpeed::illustrative` (1e-6 ms per - /// CPU operation, i.e. 1e9 operations per core-second; 1e-7 ms per - /// scanned byte): - /// - /// * `cost_per_cpu_op` = $0.000011244 per vCPU-second × - /// `ms_per_cpu_op` / 1,000 (1e-9 s per operation) = $1.1244e-14. - /// * `cost_per_scanned_byte` = $0.000011244 per vCPU-second × - /// `ms_per_scanned_byte` / 1,000 (1e-10 s per byte) = $1.1244e-15: - /// Fargate has no scan price, but a scan occupies the vCPU-seconds it - /// bills, for the time latency charges. - /// * `cost_per_memory_byte_second` = $0.000001235 per GB-second ÷ 2^30 - /// bytes per GB ≈ $1.1502e-15: the page says "on this page GB = - /// 1024^3 bytes". - /// * `cost_per_disk_byte_second` = 0: Fargate bills only ephemeral - /// storage configured beyond the 20 GB every task gets, which a - /// per-byte price cannot express, and the disk bytes plan selection - /// counts (the raw data query-time scans read) are held by the data - /// source, not by the task. - /// * `cost_per_provisioned_memory_byte_second` and - /// `cost_per_provisioned_cpu_core_second` = 0: they belong to cost - /// model 2, and Fargate's per-task billing is cost model 1. Under - /// [`CostModel::ProvisionedCapacity`] every price this preset gives - /// is 0, which [`CostInputs::validate`] refuses. - /// - /// Not expressed by these per-second prices: the 1-minute billing - /// minimum per task ("Pricing is calculated per second with a 1-minute - /// minimum"), and the fixed task sizes a task is billed for (from 0.25 - /// vCPU with 0.5 GB up), which round usage up to the next size. - pub fn aws_fargate_us_east_1(speed: ResourceSpeed) -> Self { - // US dollars per vCPU-second and per GB-second of memory, from the - // page above. - const DOLLARS_PER_VCPU_SECOND: f64 = 0.000_011_244; - const DOLLARS_PER_MEMORY_GB_SECOND: f64 = 0.000_001_235; - const BYTES_PER_GIB: f64 = (1u64 << 30) as f64; - Self { - cost_per_cpu_op: DOLLARS_PER_VCPU_SECOND * speed.ms_per_cpu_op / 1_000.0, - cost_per_scanned_byte: DOLLARS_PER_VCPU_SECOND * speed.ms_per_scanned_byte / 1_000.0, - cost_per_memory_byte_second: DOLLARS_PER_MEMORY_GB_SECOND / BYTES_PER_GIB, - cost_per_provisioned_memory_byte_second: 0.0, - cost_per_provisioned_cpu_core_second: 0.0, - cost_per_disk_byte_second: 0.0, - } - } -} - -/// How fast one core does work; the latency check reads it. -#[derive(Debug, Clone, Copy, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ResourceSpeed { - /// Milliseconds one CPU operation takes on one core. - pub ms_per_cpu_op: f64, - /// Milliseconds scanning one byte takes on one core. - pub ms_per_scanned_byte: f64, -} - -impl ResourceSpeed { - /// 1 ns per operation and 0.1 ns per scanned byte, the times - /// [`ResourceUnitCost::illustrative`] prices: one cost unit is one - /// millisecond of one core. - pub(crate) fn illustrative() -> Self { - Self { - ms_per_cpu_op: 1e-6, - ms_per_scanned_byte: 1e-7, - } - } -} - -/// How plan selection turns each valid candidate's [`ResourceUsage`] into a -/// choice. Chosen by the deployment ([`CostInputs::cost_model`]). -/// In JSON, `{"usage_over_time": {...}}`, `{"provisioned_capacity": {...}}` -/// or `{"cluster_capacity": {...}}`. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum CostModel { - /// Cost model 1, pay-as-you-go: Σ unit cost × usage per second (CPU - /// operations, scanned bytes, memory byte-seconds, disk byte-seconds); - /// the cheapest valid candidate wins. - UsageOverTime(ResourceUnitCost), - /// Cost model 2, provisioned: unit cost × peak memory, peak CPU cores - /// and disk, per second; the cheapest valid candidate wins. - ProvisionedCapacity(ResourceUnitCost), - /// Cost model 3, the deployment's own cluster: no prices; memory, disk - /// and (when set) CPU cores are hard limits, and the objective chooses - /// among the candidates that fit. - ClusterCapacity(ClusterLimits), -} - -/// A fixed cluster's capacity (cost model 3) and how to choose within it. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ClusterLimits { - /// Memory available; a candidate whose peak memory is above it is invalid. - pub memory_bytes: u64, - /// Disk available; a candidate keeping more on disk is invalid. - pub disk_bytes: u64, - /// CPU cores available; `None` leaves cores unchecked. - pub cpu_cores: Option, - /// How to choose among the candidates that fit. - pub objective: ClusterObjective, -} - -/// What cost model 3 optimizes among the candidates that fit. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ClusterObjective { - /// The smallest worst-query latency; ties by peak memory, then disk. - LowestLatency, - /// The smallest peak memory; ties by disk, then worst-query latency. - LeastMemory, -} - -/// The deployment's choices for plan selection (Stage 3): the cost model, -/// how fast a core works, and how long one-off queries are spread over. -/// Loadable from JSON (`asap_planner --cost-inputs `); -/// an omitted field takes its default, an unknown one is refused. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct CostInputs { - /// How candidates are compared; default cost model 1 with - /// [`ResourceUnitCost::illustrative`]. - pub cost_model: CostModel, - /// Time per operation and per scanned byte, for latency and cores; - /// default `ResourceSpeed::illustrative`. - pub resource_speed: ResourceSpeed, - /// Seconds one-off, scheduled and unknown queries are spread over - /// (`invocations / horizon` per second); default 3,600. - pub non_repeating_query_horizon_seconds: f64, - /// The deployment's measured CPU time per unit of each operator; a node - /// with an entry is priced from it, every other node from the formulas - /// scaled by the calibration factor (module doc, "Measured operation - /// costs"). Also loads the earlier array of [`MeasuredOperationCost`]. - /// Default empty: every node is priced from the formulas. - pub measured_operation_costs: MeasuredOperationCosts, - /// Externally established formula-to-measurement scale, held constant across - /// candidates. Must be finite and positive. None fits the complete generated - /// candidate catalog before pricing, preserving the default calibration policy. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub frozen_calibration_factor: Option, -} - -impl Default for CostInputs { - fn default() -> Self { - Self { - cost_model: CostModel::UsageOverTime(ResourceUnitCost::illustrative()), - resource_speed: ResourceSpeed::illustrative(), - non_repeating_query_horizon_seconds: DEFAULT_NON_REPEATING_QUERY_HORIZON_SECONDS, - measured_operation_costs: MeasuredOperationCosts::default(), - frozen_calibration_factor: None, - } - } -} - -impl CostInputs { - /// Bad input: a negative or non-finite speed, a horizon that is not - /// positive, a bad measured table ([`MeasuredOperationCosts`]: a - /// negative or non-finite time or size, an entry naming two families, - /// a duplicate entry, or `ms_per_cpu_op` 0), or a price the chosen cost model reads that is negative or - /// not finite, or every such price 0 (cost model 3 reads none). - pub fn validate(&self) -> Result<(), AnalyticalCostError> { - if let Some(factor) = self.frozen_calibration_factor { - CalibrationFit::from_factor(factor)?; - } - for (name, value) in [ - ("ms_per_cpu_op", self.resource_speed.ms_per_cpu_op), - ( - "ms_per_scanned_byte", - self.resource_speed.ms_per_scanned_byte, - ), - ] { - if !value.is_finite() || value < 0.0 { - return Err(AnalyticalCostError::InvalidCalibration(name, value)); - } - } - let horizon = self.non_repeating_query_horizon_seconds; - if !horizon.is_finite() || horizon <= 0.0 { - return Err(AnalyticalCostError::InvalidCalibration( - "non_repeating_query_horizon_seconds", - horizon, - )); - } - measured::validate( - &self.measured_operation_costs, - self.resource_speed.ms_per_cpu_op, - )?; - let read = match &self.cost_model { - CostModel::UsageOverTime(p) => vec![ - ("cost_per_cpu_op", p.cost_per_cpu_op), - ("cost_per_scanned_byte", p.cost_per_scanned_byte), - ("cost_per_memory_byte_second", p.cost_per_memory_byte_second), - ("cost_per_disk_byte_second", p.cost_per_disk_byte_second), - ], - CostModel::ProvisionedCapacity(p) => vec![ - ( - "cost_per_provisioned_memory_byte_second", - p.cost_per_provisioned_memory_byte_second, - ), - ( - "cost_per_provisioned_cpu_core_second", - p.cost_per_provisioned_cpu_core_second, - ), - ("cost_per_disk_byte_second", p.cost_per_disk_byte_second), - ], - CostModel::ClusterCapacity(_) => return Ok(()), - }; - for &(name, value) in &read { - if !value.is_finite() || value < 0.0 { - return Err(AnalyticalCostError::InvalidCalibration(name, value)); - } - } - if read.iter().all(|(_, value)| *value == 0.0) { - return Err(AnalyticalCostError::ZeroCalibration); - } - Ok(()) - } -} - -/// One node's price per second of wall time, with how it was measured. A -/// node is one operator node of the candidate's physical plan DAG -/// (`PhysicalASAPDAG`), keyed by its [`PhysicalASAPNodeId`] in -/// [`PlanCost::per_node`]; not a machine. -#[derive(Debug, Clone, PartialEq)] -pub struct NodeCost { - /// Its operator, as the measured table names it. - pub operator: OperatorKind, - /// When it runs: ingestion or query time. - pub timing: ExecutionTiming, - /// Its price per second under cost model 1; `None` under cost models 2 - /// and 3, which price (or limit) the whole candidate's peaks. - pub cost_per_second: Option, - /// Core-seconds of CPU it takes per second of wall time: its CPU - /// operations and scanned bytes timed by [`ResourceSpeed`], what a - /// runtime measures it against. A query's fixed per-request cost - /// ([`MeasuredOperationCosts::ns_per_evaluation`]) is counted on the - /// node the query reads first. - pub cpu_core_seconds_per_second: f64, - /// The number of consumers it is computed for, with the - /// `MatAtIngestion` consumers of one refresh window counted once (1 - /// when materialized, [`execution_demand::computations`]). - pub multiplier: u64, - /// How it was measured, e.g. "scan 1000 samples; x 1.0000/s; keeps - /// ... bytes": one computation's work, the times it is computed per - /// second and what it keeps. - pub detail: String, - /// Whether its CPU operations were priced from the measured table or - /// from the formulas. - pub price_source: NodePriceSource, -} - -/// One candidate's usage and price over the whole workload: `usage` is -/// the whole candidate's, `per_node` one price per operator node of its -/// physical plan DAG. -#[derive(Debug, Clone, PartialEq)] -pub struct PlanCost { - pub usage: ResourceUsage, - /// Its price per second under cost model 1 or 2 (under cost model 1, the - /// sum of its nodes' charges plus request overhead for queries without - /// operator nodes); `None` under cost model 3. - pub cost_per_second: Option, - /// The unit of `cost_per_second`: always [`COST_UNIT`]. - pub cost_unit: &'static str, - /// Where its prices came from, from its nodes' [`NodeCost::price_source`]: - /// [`COST_SOURCE`] (all formulas), [`MEASURED_COST_SOURCE`] (all - /// measured) or [`MIXED_COST_SOURCE`]. - pub source: &'static str, - /// Each operator node of the candidate's physical plan DAG, by its id - /// in that DAG, with its price. - pub per_node: BTreeMap, -} - -/// One query's latency per evaluation, its longest chain of work (module -/// doc, "Latency"): the compaction of the newest window of a state it reads -/// from ingestion time, then the query's own work, each on a full core. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct QueryLatency { - /// Milliseconds to merge the ingest workers' partial copies of the - /// newest window of the largest ingestion-time state the query reads; - /// 0 when ingestion fits on one core or the query reads none. - pub compaction_ms: f64, - /// Milliseconds of the query-time work the query reaches, each node - /// once. - pub query_ms: f64, -} - -impl QueryLatency { - /// `compaction_ms + query_ms`: what is checked against the query's - /// `max_latency_ms`. - pub fn total_ms(&self) -> f64 { - self.compaction_ms + self.query_ms - } -} - -/// The CPU time of the nodes priced from the measured table, per second of -/// wall time: as measured, and as the formulas would price them. Their -/// ratio is the calibration factor for the nodes without an entry. -#[derive(Debug, Clone, Copy, Default, PartialEq)] -pub struct CalibrationFit { - measured_ms_per_second: f64, - formula_ms_per_second: f64, -} - -impl CalibrationFit { - /// A validated externally established scale, without fitting candidate data. - pub fn from_factor(factor: f64) -> Result { - if !factor.is_finite() || factor <= 0.0 { - return Err(AnalyticalCostError::InvalidCalibration( - "frozen_calibration_factor", - factor, - )); - } - Ok(Self { - measured_ms_per_second: factor, - formula_ms_per_second: 1.0, - }) - } - - /// Record one measured node: its measured and formula CPU time per - /// second of wall time. - pub fn record(&mut self, measured_ms_per_second: f64, formula_ms_per_second: f64) { - self.measured_ms_per_second += measured_ms_per_second; - self.formula_ms_per_second += formula_ms_per_second; - } - - /// Add `other`'s CPU time to this fit. - pub fn add(&mut self, other: CalibrationFit) { - self.measured_ms_per_second += other.measured_ms_per_second; - self.formula_ms_per_second += other.formula_ms_per_second; - } - - /// Measured ÷ formula time, 1 without measured nodes (or with only - /// nodes the formulas price free). - pub fn factor(self) -> f64 { - match self.formula_ms_per_second > 0.0 && self.measured_ms_per_second > 0.0 { - true => self.measured_ms_per_second / self.formula_ms_per_second, - false => 1.0, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The AWS Fargate preset holds the page's prices converted as its doc - /// says at the given speed: per vCPU-second × seconds per operation or - /// per scanned byte (1e-9 and 1e-10 at the illustrative speed; twice - /// that at half the speed), per GB-second ÷ 2^30; disk and the - /// provisioned prices 0. It validates under cost model 1 and is - /// refused under cost model 2, where every price it gives is 0. - #[test] - fn aws_fargate_us_east_1_converts_the_page_prices() { - let preset = ResourceUnitCost::aws_fargate_us_east_1(ResourceSpeed::illustrative()); - let close = |actual: f64, expected: f64| { - assert!( - ((actual - expected) / expected).abs() < 1e-12, - "{actual} != {expected}" - ) - }; - close(preset.cost_per_cpu_op, 1.1244e-14); - close(preset.cost_per_scanned_byte, 1.1244e-15); - close( - preset.cost_per_memory_byte_second, - 1.235e-6 / 1_073_741_824.0, - ); - assert_eq!(preset.cost_per_disk_byte_second, 0.0); - assert_eq!(preset.cost_per_provisioned_memory_byte_second, 0.0); - assert_eq!(preset.cost_per_provisioned_cpu_core_second, 0.0); - // A core twice as slow takes twice the vCPU-seconds per operation and per byte. - let slower = ResourceUnitCost::aws_fargate_us_east_1(ResourceSpeed { - ms_per_cpu_op: 2e-6, - ms_per_scanned_byte: 2e-7, - }); - close(slower.cost_per_cpu_op, 2.2488e-14); - close(slower.cost_per_scanned_byte, 2.2488e-15); - close( - slower.cost_per_memory_byte_second, - preset.cost_per_memory_byte_second, - ); - let settings = |cost_model| CostInputs { - cost_model, - ..CostInputs::default() - }; - assert_eq!( - settings(CostModel::UsageOverTime(preset.clone())).validate(), - Ok(()) - ); - assert_eq!( - settings(CostModel::ProvisionedCapacity(preset)).validate(), - Err(AnalyticalCostError::ZeroCalibration) - ); - } -} +pub use node::operator_statistics::EdgeStatistics; +pub use node::usage::{ + node_map, node_usage, summary_type_name, NodeCost, NodeMap, NodeUsage, ProducerOutput, +}; +pub use plan::cost::{plan_cost, PlanCost, QueryLatency}; +pub use plan::ranking::{cheaper, worst_latency_ms, PlanRanking}; +pub use plan::retention::{ + maintained_window_ms, query_reach, retained_tumbling_windows, tumbling_window_retention_ms, + tumbling_window_retention_ms_of_plan, tumbling_windows_of, +}; +pub use resources::ResourceUsage; diff --git a/crates/cost/src/measured.rs b/crates/cost/src/measured.rs index 1fabd322c..c587526a6 100644 --- a/crates/cost/src/measured.rs +++ b/crates/cost/src/measured.rs @@ -3,9 +3,9 @@ //! from instead of its formulas. //! //! The Planner defines the table's structure; the deployment's runtime -//! measures the numbers (e.g. ASAPQuery-backend's `measure_costs`, or -//! sketch-bench's `rqe_atomic_costs.json` converted by `asap-devtools`' -//! `convert_rqe_atomic_costs`) and puts the table in +//! measures the numbers (e.g. sketch-bench's `rqe_atomic_costs.json` +//! converted by `asap-devtools`' `convert_rqe_atomic_costs`) and puts the +//! table in //! [`CostInputs::measured_operation_costs`]. The planner reads //! only this table, never a benchmark file. //! @@ -38,13 +38,13 @@ use std::mem::discriminant; use asap_types::ir::operator::AggIntent; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload as Payload; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::schema::{ExactKind, SketchParams}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload as Payload; +use asap_types::summary_type::{ExactKind, SketchParams}; use serde::{Deserialize, Serialize}; -use crate::analytical::AnalyticalCostError; +use crate::node::analytical::AnalyticalCostError; /// The deployment's measured cost table /// ([`CostInputs::measured_operation_costs`]). In JSON, an object @@ -503,14 +503,14 @@ struct TableFields { provenance: Option, } -/// Which prices one node's CPU operations came from. A node is one operator +/// Which costs one node's CPU operations came from. A node is one operator /// node of the candidate's physical plan DAG (`PhysicalASAPDAG`), keyed by /// its `PhysicalASAPNodeId` in [`PlanCost::per_node`]; not a machine. /// /// [`PlanCost::per_node`]: crate::PlanCost::per_node #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] -pub enum NodePriceSource { +pub enum NodeCostSource { /// Plan selection's formulas (`analytical_cost` and the sketch shapes), /// scaled by the calibration factor when the table has entries. Analytical, diff --git a/crates/cost/src/model.rs b/crates/cost/src/model.rs new file mode 100644 index 000000000..caba6b63b --- /dev/null +++ b/crates/cost/src/model.rs @@ -0,0 +1,386 @@ +//! The cost models and the deployment's cost inputs: what each unit of a +//! resource costs ([`ResourceUnitCost`]), how fast each resource runs +//! ([`ResourceSpeed`]), the cluster's limits and objective, and the +//! calibration of measured costs. + +use crate::node::analytical::AnalyticalCostError; +use crate::MeasuredOperationCosts; + +/// [`PlanCost::source`](crate::PlanCost::source) when every node is priced from the formulas. +pub const COST_SOURCE: &str = "analytical-cost-v2 (illustrative statistics)"; + +/// [`PlanCost::source`](crate::PlanCost::source) when every node is priced from the measured +/// table. +pub const MEASURED_COST_SOURCE: &str = "measured operation costs (illustrative statistics)"; + +/// [`PlanCost::source`](crate::PlanCost::source) when some nodes are priced from the measured +/// table and the others from the formulas. +pub const MIXED_COST_SOURCE: &str = + "measured operation costs and analytical-cost-v2 (illustrative statistics)"; + +/// Seconds over which non-repeating queries (one-off, scheduled and +/// unknown) are spread by default: one hour. +pub(crate) const DEFAULT_NON_REPEATING_QUERY_HORIZON_SECONDS: f64 = 3_600.0; + +/// The price of one unit of each resource, per second of wall time. Cost +/// model 1 reads the usage prices (CPU, scan, memory byte-seconds, disk); +/// cost model 2 the provisioned prices (memory, CPU cores) and disk. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ResourceUnitCost { + /// Price of one CPU operation. + pub cost_per_cpu_op: f64, + /// Price of scanning one byte. + pub cost_per_scanned_byte: f64, + /// Price of holding one byte of memory for one second (cost model 1). + pub cost_per_memory_byte_second: f64, + /// Price of provisioning one byte of memory for one second (cost model 2). + pub cost_per_provisioned_memory_byte_second: f64, + /// Price of provisioning one CPU core for one second (cost model 2). + pub cost_per_provisioned_cpu_core_second: f64, + /// Price of keeping one byte on disk for one second (both models). + pub cost_per_disk_byte_second: f64, +} + +impl ResourceUnitCost { + /// The prices plan selection uses until a deployment is calibrated: + /// 1 ns of CPU per operation, 0.1 ns per scanned byte, 1 GB retained for a + /// second costs 1/8 vCPU (125 CPU-ms), held or provisioned alike; one + /// provisioned core-second is 1,000 CPU-ms. Disk is not priced (0). + pub fn illustrative() -> Self { + Self { + cost_per_cpu_op: 1e-6, + cost_per_scanned_byte: 1e-7, + cost_per_memory_byte_second: 1.25e-7, + cost_per_provisioned_memory_byte_second: 1.25e-7, + cost_per_provisioned_cpu_core_second: 1_000.0, + cost_per_disk_byte_second: 0.0, + } + } + + /// AWS Fargate on-demand prices, US East (N. Virginia), Linux/x86, for + /// cost model 1 ([`CostModel::UsageOverTime`]); one cost unit is one + /// US dollar. Source: , checked + /// 2026-10-18 (Pricing Example 1 there: $0.000011244 per vCPU-second, + /// $0.000001235 per GB-second of memory, $0.0000000308 per GB-second of + /// ephemeral storage beyond the 20 GB included). Prices change: re-check + /// the page before relying on them. + /// + /// `speed` is the deployment's [`CostInputs::resource_speed`], + /// the speed Stage 3 uses for latency, so the CPU prices charge the core + /// time latency counts. At `ResourceSpeed::illustrative` (1e-6 ms per + /// CPU operation, i.e. 1e9 operations per core-second; 1e-7 ms per + /// scanned byte): + /// + /// * `cost_per_cpu_op` = $0.000011244 per vCPU-second × + /// `ms_per_cpu_op` / 1,000 (1e-9 s per operation) = $1.1244e-14. + /// * `cost_per_scanned_byte` = $0.000011244 per vCPU-second × + /// `ms_per_scanned_byte` / 1,000 (1e-10 s per byte) = $1.1244e-15: + /// Fargate has no scan price, but a scan occupies the vCPU-seconds it + /// bills, for the time latency charges. + /// * `cost_per_memory_byte_second` = $0.000001235 per GB-second ÷ 2^30 + /// bytes per GB ≈ $1.1502e-15: the page says "on this page GB = + /// 1024^3 bytes". + /// * `cost_per_disk_byte_second` = 0: Fargate bills only ephemeral + /// storage configured beyond the 20 GB every task gets, which a + /// per-byte price cannot express, and the disk bytes plan selection + /// counts (the raw data query-time scans read) are held by the data + /// source, not by the task. + /// * `cost_per_provisioned_memory_byte_second` and + /// `cost_per_provisioned_cpu_core_second` = 0: they belong to cost + /// model 2, and Fargate's per-task billing is cost model 1. Under + /// [`CostModel::ProvisionedCapacity`] every price this preset gives + /// is 0, which [`CostInputs::validate`] refuses. + /// + /// Not expressed by these per-second prices: the 1-minute billing + /// minimum per task ("Pricing is calculated per second with a 1-minute + /// minimum"), and the fixed task sizes a task is billed for (from 0.25 + /// vCPU with 0.5 GB up), which round usage up to the next size. + pub fn aws_fargate_us_east_1(speed: ResourceSpeed) -> Self { + // US dollars per vCPU-second and per GB-second of memory, from the + // page above. + const DOLLARS_PER_VCPU_SECOND: f64 = 0.000_011_244; + const DOLLARS_PER_MEMORY_GB_SECOND: f64 = 0.000_001_235; + const BYTES_PER_GIB: f64 = (1u64 << 30) as f64; + Self { + cost_per_cpu_op: DOLLARS_PER_VCPU_SECOND * speed.ms_per_cpu_op / 1_000.0, + cost_per_scanned_byte: DOLLARS_PER_VCPU_SECOND * speed.ms_per_scanned_byte / 1_000.0, + cost_per_memory_byte_second: DOLLARS_PER_MEMORY_GB_SECOND / BYTES_PER_GIB, + cost_per_provisioned_memory_byte_second: 0.0, + cost_per_provisioned_cpu_core_second: 0.0, + cost_per_disk_byte_second: 0.0, + } + } +} + +/// How fast one core does work; the latency check reads it. +#[derive(Debug, Clone, Copy, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ResourceSpeed { + /// Milliseconds one CPU operation takes on one core. + pub ms_per_cpu_op: f64, + /// Milliseconds scanning one byte takes on one core. + pub ms_per_scanned_byte: f64, +} + +impl ResourceSpeed { + /// 1 ns per operation and 0.1 ns per scanned byte, the times + /// [`ResourceUnitCost::illustrative`] prices: one cost unit is one + /// millisecond of one core. + pub(crate) fn illustrative() -> Self { + Self { + ms_per_cpu_op: 1e-6, + ms_per_scanned_byte: 1e-7, + } + } +} + +/// How plan selection turns each valid candidate's [`ResourceUsage`](crate::ResourceUsage) into a +/// choice. Chosen by the deployment ([`CostInputs::cost_model`]). +/// In JSON, `{"usage_over_time": {...}}`, `{"provisioned_capacity": {...}}` +/// or `{"cluster_capacity": {...}}`. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CostModel { + /// Cost model 1, pay-as-you-go: Σ unit cost × usage per second (CPU + /// operations, scanned bytes, memory byte-seconds, disk byte-seconds); + /// the cheapest valid candidate wins. + UsageOverTime(ResourceUnitCost), + /// Cost model 2, provisioned: unit cost × peak memory, peak CPU cores + /// and disk, per second; the cheapest valid candidate wins. + ProvisionedCapacity(ResourceUnitCost), + /// Cost model 3, the deployment's own cluster: no prices; memory, disk + /// and (when set) CPU cores are hard limits, and the objective chooses + /// among the candidates that fit. + ClusterCapacity(ClusterLimits), +} + +/// A fixed cluster's capacity (cost model 3) and how to choose within it. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClusterLimits { + /// Memory available; a candidate whose peak memory is above it is invalid. + pub memory_bytes: u64, + /// Disk available; a candidate keeping more on disk is invalid. + pub disk_bytes: u64, + /// CPU cores available; `None` leaves cores unchecked. + pub cpu_cores: Option, + /// How to choose among the candidates that fit. + pub objective: ClusterObjective, +} + +/// What cost model 3 optimizes among the candidates that fit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ClusterObjective { + /// The smallest worst-query latency; ties by peak memory, then disk. + LowestLatency, + /// The smallest peak memory; ties by disk, then worst-query latency. + LeastMemory, +} + +/// The deployment's choices for plan selection (Stage 3): the cost model, +/// how fast a core works, and how long one-off queries are spread over. +/// Loadable from JSON (`asap_planner --cost-inputs `); +/// an omitted field takes its default, an unknown one is refused. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct CostInputs { + /// How candidates are compared; default cost model 1 with + /// [`ResourceUnitCost::illustrative`]. + pub cost_model: CostModel, + /// Time per operation and per scanned byte, for latency and cores; + /// default `ResourceSpeed::illustrative`. + pub resource_speed: ResourceSpeed, + /// Seconds one-off, scheduled and unknown queries are spread over + /// (`invocations / horizon` per second); default 3,600. + pub non_repeating_query_horizon_seconds: f64, + /// The deployment's measured CPU time per unit of each operator; a node + /// with an entry is priced from it, every other node from the formulas + /// scaled by the calibration factor (module doc, "Measured operation + /// costs"). Also loads the earlier array of [`MeasuredOperationCost`](crate::MeasuredOperationCost). + /// Default empty: every node is priced from the formulas. + pub measured_operation_costs: MeasuredOperationCosts, + /// Externally established formula-to-measurement scale, held constant across + /// candidates. Must be finite and positive. None fits the complete generated + /// candidate catalog before pricing, preserving the default calibration policy. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub frozen_calibration_factor: Option, +} + +impl Default for CostInputs { + fn default() -> Self { + Self { + cost_model: CostModel::UsageOverTime(ResourceUnitCost::illustrative()), + resource_speed: ResourceSpeed::illustrative(), + non_repeating_query_horizon_seconds: DEFAULT_NON_REPEATING_QUERY_HORIZON_SECONDS, + measured_operation_costs: MeasuredOperationCosts::default(), + frozen_calibration_factor: None, + } + } +} + +impl CostInputs { + /// Bad input: a negative or non-finite speed, a horizon that is not + /// positive, a bad measured table ([`MeasuredOperationCosts`]: a + /// negative or non-finite time or size, an entry naming two families, + /// a duplicate entry, or `ms_per_cpu_op` 0), or a price the chosen cost model reads that is negative or + /// not finite, or every such price 0 (cost model 3 reads none). + pub fn validate(&self) -> Result<(), AnalyticalCostError> { + if let Some(factor) = self.frozen_calibration_factor { + CalibrationFit::from_factor(factor)?; + } + for (name, value) in [ + ("ms_per_cpu_op", self.resource_speed.ms_per_cpu_op), + ( + "ms_per_scanned_byte", + self.resource_speed.ms_per_scanned_byte, + ), + ] { + if !value.is_finite() || value < 0.0 { + return Err(AnalyticalCostError::InvalidCalibration(name, value)); + } + } + let horizon = self.non_repeating_query_horizon_seconds; + if !horizon.is_finite() || horizon <= 0.0 { + return Err(AnalyticalCostError::InvalidCalibration( + "non_repeating_query_horizon_seconds", + horizon, + )); + } + crate::measured::validate( + &self.measured_operation_costs, + self.resource_speed.ms_per_cpu_op, + )?; + let read = match &self.cost_model { + CostModel::UsageOverTime(p) => vec![ + ("cost_per_cpu_op", p.cost_per_cpu_op), + ("cost_per_scanned_byte", p.cost_per_scanned_byte), + ("cost_per_memory_byte_second", p.cost_per_memory_byte_second), + ("cost_per_disk_byte_second", p.cost_per_disk_byte_second), + ], + CostModel::ProvisionedCapacity(p) => vec![ + ( + "cost_per_provisioned_memory_byte_second", + p.cost_per_provisioned_memory_byte_second, + ), + ( + "cost_per_provisioned_cpu_core_second", + p.cost_per_provisioned_cpu_core_second, + ), + ("cost_per_disk_byte_second", p.cost_per_disk_byte_second), + ], + CostModel::ClusterCapacity(_) => return Ok(()), + }; + for &(name, value) in &read { + if !value.is_finite() || value < 0.0 { + return Err(AnalyticalCostError::InvalidCalibration(name, value)); + } + } + if read.iter().all(|(_, value)| *value == 0.0) { + return Err(AnalyticalCostError::ZeroCalibration); + } + Ok(()) + } +} + +/// The CPU time of the nodes priced from the measured table, per second of +/// wall time: as measured, and as the formulas would price them. Their +/// ratio is the calibration factor for the nodes without an entry. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct CalibrationFit { + measured_ms_per_second: f64, + formula_ms_per_second: f64, +} + +impl CalibrationFit { + /// A validated externally established scale, without fitting candidate data. + pub fn from_factor(factor: f64) -> Result { + if !factor.is_finite() || factor <= 0.0 { + return Err(AnalyticalCostError::InvalidCalibration( + "frozen_calibration_factor", + factor, + )); + } + Ok(Self { + measured_ms_per_second: factor, + formula_ms_per_second: 1.0, + }) + } + + /// Record one measured node: its measured and formula CPU time per + /// second of wall time. + pub fn record(&mut self, measured_ms_per_second: f64, formula_ms_per_second: f64) { + self.measured_ms_per_second += measured_ms_per_second; + self.formula_ms_per_second += formula_ms_per_second; + } + + /// Add `other`'s CPU time to this fit. + pub fn add(&mut self, other: CalibrationFit) { + self.measured_ms_per_second += other.measured_ms_per_second; + self.formula_ms_per_second += other.formula_ms_per_second; + } + + /// Measured ÷ formula time, 1 without measured nodes (or with only + /// nodes the formulas price free). + pub fn factor(self) -> f64 { + match self.formula_ms_per_second > 0.0 && self.measured_ms_per_second > 0.0 { + true => self.measured_ms_per_second / self.formula_ms_per_second, + false => 1.0, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The AWS Fargate preset holds the page's prices converted as its doc + /// says at the given speed: per vCPU-second × seconds per operation or + /// per scanned byte (1e-9 and 1e-10 at the illustrative speed; twice + /// that at half the speed), per GB-second ÷ 2^30; disk and the + /// provisioned prices 0. It validates under cost model 1 and is + /// refused under cost model 2, where every price it gives is 0. + #[test] + fn aws_fargate_us_east_1_converts_the_page_prices() { + let preset = ResourceUnitCost::aws_fargate_us_east_1(ResourceSpeed::illustrative()); + let close = |actual: f64, expected: f64| { + assert!( + ((actual - expected) / expected).abs() < 1e-12, + "{actual} != {expected}" + ) + }; + close(preset.cost_per_cpu_op, 1.1244e-14); + close(preset.cost_per_scanned_byte, 1.1244e-15); + close( + preset.cost_per_memory_byte_second, + 1.235e-6 / 1_073_741_824.0, + ); + assert_eq!(preset.cost_per_disk_byte_second, 0.0); + assert_eq!(preset.cost_per_provisioned_memory_byte_second, 0.0); + assert_eq!(preset.cost_per_provisioned_cpu_core_second, 0.0); + // A core twice as slow takes twice the vCPU-seconds per operation and per byte. + let slower = ResourceUnitCost::aws_fargate_us_east_1(ResourceSpeed { + ms_per_cpu_op: 2e-6, + ms_per_scanned_byte: 2e-7, + }); + close(slower.cost_per_cpu_op, 2.2488e-14); + close(slower.cost_per_scanned_byte, 2.2488e-15); + close( + slower.cost_per_memory_byte_second, + preset.cost_per_memory_byte_second, + ); + let settings = |cost_model| CostInputs { + cost_model, + ..CostInputs::default() + }; + assert_eq!( + settings(CostModel::UsageOverTime(preset.clone())).validate(), + Ok(()) + ); + assert_eq!( + settings(CostModel::ProvisionedCapacity(preset)).validate(), + Err(AnalyticalCostError::ZeroCalibration) + ); + } +} diff --git a/crates/cost/src/analytical.rs b/crates/cost/src/node/analytical.rs similarity index 98% rename from crates/cost/src/analytical.rs rename to crates/cost/src/node/analytical.rs index 6c6640a18..e476dea25 100644 --- a/crates/cost/src/analytical.rs +++ b/crates/cost/src/node/analytical.rs @@ -5,10 +5,10 @@ //! [`crate::ResourceUnitCost`]. use crate::resources::ModeledCpu; -use asap_types::ir::schema::{SketchAlgorithm, SketchParams}; +use asap_types::summary_type::{SketchAlgorithm, SketchParams}; use serde::{Deserialize, Serialize}; -use crate::operator_statistics::{ +use crate::node::operator_statistics::{ EdgeStatistics, OperatorStatistics, PromqlEdgeStatistics, PromqlValueKind, }; @@ -302,7 +302,7 @@ fn checked_cpu_product(rows: u64, operations_per_row: u64) -> Result Result { @@ -323,7 +323,7 @@ fn partitioned_order_estimate( fn validate_partitioning( input: EdgeStatistics, - partitioning: &crate::operator_statistics::PartitionStatistics, + partitioning: &crate::node::operator_statistics::PartitionStatistics, partitioned: bool, ) -> Result<(), AnalyticalCostError> { let inconsistent = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); @@ -1224,8 +1224,8 @@ pub(crate) fn validate_operator_semantics( } fn require_promql_unary( - edges: &crate::operator_statistics::UnaryEdgeStatistics, -) -> Result { + edges: &crate::node::operator_statistics::UnaryEdgeStatistics, +) -> Result { let promql = edges.promql.ok_or(AnalyticalCostError::MissingOrStale( "promql_edge_statistics", ))?; @@ -1235,8 +1235,8 @@ fn require_promql_unary( } fn require_promql_binary( - edges: &crate::operator_statistics::BinaryEdgeStatistics, -) -> Result { + edges: &crate::node::operator_statistics::BinaryEdgeStatistics, +) -> Result { let promql = edges.promql.ok_or(AnalyticalCostError::MissingOrStale( "promql_edge_statistics", ))?; @@ -1247,7 +1247,7 @@ fn require_promql_binary( } fn validate_promql_cardinality_preserving_shape( - promql: crate::operator_statistics::PromqlUnaryEdgeStatistics, + promql: crate::node::operator_statistics::PromqlUnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { validate_promql_edge(promql.input)?; validate_promql_edge(promql.output)?; @@ -1261,7 +1261,7 @@ fn validate_promql_cardinality_preserving_shape( } fn validate_promql_filter_shape( - promql: crate::operator_statistics::PromqlUnaryEdgeStatistics, + promql: crate::node::operator_statistics::PromqlUnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { validate_promql_edge(promql.input)?; validate_promql_edge(promql.output)?; @@ -1327,7 +1327,7 @@ fn validate_promql_edge_shape( fn validate_promql_bridge( operator: PhysicalOperator, - edges: &crate::operator_statistics::UnaryEdgeStatistics, + edges: &crate::node::operator_statistics::UnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { let promql = require_promql_unary(edges)?; let valid = match operator { @@ -1361,7 +1361,7 @@ fn validate_promql_binary( cardinality: PromqlVectorCardinality, build_side: Option, matching_key_bytes: u64, - edges: &crate::operator_statistics::BinaryEdgeStatistics, + edges: &crate::node::operator_statistics::BinaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { let promql = require_promql_binary(edges)?; validate_instant_vector_rows(edges.output, promql.output)?; @@ -1430,7 +1430,7 @@ fn validate_promql_series_sample( grouping_key_count: u64, group_count: u64, key_bytes: u64, - edges: &crate::operator_statistics::UnaryEdgeStatistics, + edges: &crate::node::operator_statistics::UnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { let promql = require_promql_unary(edges)?; validate_instant_vector_rows(edges.output, promql.output)?; @@ -1536,7 +1536,7 @@ fn checked_bytes(parts: &[u64]) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::operator_statistics::{ + use crate::node::operator_statistics::{ BinaryEdgeStatistics, EdgeStatistics, OperatorStatistics, PartitionStatistics, PromqlBinaryEdgeStatistics, PromqlEdgeStatistics, PromqlUnaryEdgeStatistics, PromqlValueKind, UnaryEdgeStatistics, diff --git a/crates/cost/src/node/mod.rs b/crates/cost/src/node/mod.rs new file mode 100644 index 000000000..1ab5a2605 --- /dev/null +++ b/crates/cost/src/node/mod.rs @@ -0,0 +1,9 @@ +//! One node of a physical plan DAG: its resource usage and cost +//! (`usage`), from the analytical per-operator resource model +//! ([`analytical`], `operator_statistics`) and the bytes a state holds +//! ([`state_bytes`]). + +pub mod analytical; +pub(crate) mod operator_statistics; +pub mod state_bytes; +pub(crate) mod usage; diff --git a/crates/cost/src/operator_statistics.rs b/crates/cost/src/node/operator_statistics.rs similarity index 99% rename from crates/cost/src/operator_statistics.rs rename to crates/cost/src/node/operator_statistics.rs index 796073573..f382dec0f 100644 --- a/crates/cost/src/operator_statistics.rs +++ b/crates/cost/src/node/operator_statistics.rs @@ -96,7 +96,7 @@ pub(crate) struct PartitionStatistics { } /// Workload-dependent evidence for one operator in an already-lowered -/// physical DAG. [`PhysicalOperator`](crate::analytical::PhysicalOperator) +/// physical DAG. [`PhysicalOperator`](crate::node::analytical::PhysicalOperator) /// is the authoritative operator vocabulary: every one of its variants has a /// matching statistics variant here. /// diff --git a/crates/cost/src/state_bytes.rs b/crates/cost/src/node/state_bytes.rs similarity index 80% rename from crates/cost/src/state_bytes.rs rename to crates/cost/src/node/state_bytes.rs index a8d4c8fba..7c3d6f667 100644 --- a/crates/cost/src/state_bytes.rs +++ b/crates/cost/src/node/state_bytes.rs @@ -2,10 +2,9 @@ //! per input row, from its parameters and the rows it folds. use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::sketch::KLL_MAX_K; -use asap_types::ir::schema::{ - DataType, FieldDataType, Schema, SketchParams, EXACT_ACCUMULATOR_STATE_BYTES, -}; +use asap_types::ir::schema::{DataType, FieldDataType, Schema}; +use asap_types::summary_type::KLL_MAX_K; +use asap_types::summary_type::{SketchParams, EXACT_ACCUMULATOR_STATE_BYTES}; /// Plain values are 8 bytes, strings 16; summary columns are sized apart. pub fn row_bytes(schema: &Schema) -> u64 { @@ -60,7 +59,10 @@ pub fn keeps_every_value(measure: &AggIntent) -> bool { /// 1 for every other sketch. pub fn summary_shape(family: &FieldDataType, input_rows: StateInputRows) -> (f64, u64) { match family { - FieldDataType::Sketch(kind, _) => sketch_shape(kind.params(), input_rows), + FieldDataType::Sketch(..) => { + let _ = input_rows; + todo!("#1207 PR 6: the state shape of θ from the candidate's LogicalProperties::parameters") + } FieldDataType::ExactAggregate(..) => (1.0, EXACT_ACCUMULATOR_STATE_BYTES), _ => (1.0, 8), } @@ -104,6 +106,7 @@ pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, usize::try_from(rows.max(1)).unwrap_or(usize::MAX) }); match params { + SketchParams::Hydra(hydra) => sketch_shape(&hydra.per_subpopulation(), input_rows), SketchParams::Cms { width, depth } => ( f64::from(*depth), CountMinF64::estimated_memory_bytes(*depth as usize, *width as usize) as u64, @@ -270,14 +273,14 @@ const DDSKETCH_ASSUMED_VALUE_RANGE: f64 = 1e9; #[cfg(test)] mod tests { use super::*; - use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SketchKind}; + use asap_types::summary_type::SketchAlgorithm; /// An exact accumulator's state is priced at the executor's per-group /// state, 56 bytes for every kind (a counter's state alone is 48), so a /// tumbling-window chain of exact sums keeps 56 bytes per tumbling window and group. #[test] fn exact_state_is_priced_at_the_executors_state_size() { - use asap_types::ir::schema::{ExactKind, ExactParams}; + use asap_types::summary_type::{ExactKind, ExactParams}; for (kind, params) in [ (ExactKind::Sum, ExactParams::Sum), (ExactKind::Increase, ExactParams::Increase), @@ -291,7 +294,7 @@ mod tests { (1.0, 56) ); } - assert_eq!(asap_types::ir::schema::EXACT_ACCUMULATOR_STATE_BYTES, 56); + assert_eq!(asap_types::summary_type::EXACT_ACCUMULATOR_STATE_BYTES, 56); } /// The bytes `asap_sketchlib`'s DDSketch holds after adding `values`. @@ -353,121 +356,6 @@ mod tests { assert_eq!(priced(Some(10_000), false), priced(None, false)); } - /// DDSketch state grows as its relative accuracy α tightens. - #[test] - fn ddsketch_state_grows_as_alpha_shrinks() { - let bytes = |alpha| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .1 - }; - assert!( - bytes(0.1) < bytes(0.01), - "{} vs {}", - bytes(0.1), - bytes(0.01) - ); - assert!( - bytes(0.01) < bytes(0.001), - "{} vs {}", - bytes(0.01), - bytes(0.001) - ); - } - - /// Every sketch family's state is sized from its parameters, as - /// `asap_sketchlib` allocates it: an HLL's registers, a KMV's `k` - /// hashes once it has seen `k` distinct items, and a UnivMon's layers - /// once keys of the assumed size have churned through every heap. - #[test] - fn summary_state_is_sized_from_sketch_params() { - use asap_sketchlib::DataInput; - let bytes = |algorithm, params| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .1 - }; - assert_eq!( - bytes( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 14, - relative_error: None - } - ), - asap_sketchlib::HyperLogLog::::new().memory_bytes() as u64 - ); - let mut kmv: asap_sketchlib::Kmv = asap_sketchlib::Kmv::new(1_000); - for i in 0..2_000u64 { - kmv.update(&DataInput::U64(i)); - } - assert_eq!( - bytes( - SketchAlgorithm::Kmv, - SketchParams::Kmv { - k: 1_000, - relative_error: None - } - ), - kmv.memory_bytes() as u64 - ); - let mut univmon = asap_sketchlib::UnivMon::new(8, 3, 64, 2); - for i in 0..10_000u64 { - let mut key = [0; ASSUMED_VALUE_KEY_BYTES]; - key[..8].copy_from_slice(&i.to_le_bytes()); - univmon.update_weighted(&DataInput::Bytes(&key), 1); - } - assert_eq!( - bytes( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 8, - sketch_rows: 3, - sketch_cols: 64, - layers: 2, - distinct_failure_probability: None, - } - ), - univmon.memory_bytes() as u64 - ); - } - - /// KLL state whose samples are unknown is charged its fixed 744 bytes - /// and the item array at the capacity `asap_sketchlib` grows it to: - /// `compute_max_capacity(k, 8)` 8-byte items, with `k` capped at 26 602. - #[test] - fn kll_state_is_sized_as_asap_sketchlib_allocates_it() { - let bytes = |k| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .1 - }; - assert_eq!(bytes(269), 744 + 8 * 1_206); - assert_eq!(bytes(26_602), 744 + 8 * 80_118); - assert_eq!(bytes(30_000), 744 + 8 * 80_118); - } - /// A KLL state is charged what `asap_sketchlib` allocates for the /// samples it sees, not its full capacity: its fixed struct and level /// array, plus the item array the library starts empty and doubles from @@ -487,10 +375,7 @@ mod tests { }; let model_bytes = |rows_per_state| { sketch_shape( - &SketchParams::Kll { - k, - failure_probability: None, - }, + &SketchParams::Kll { k }, StateInputRows { rows_per_state, non_negative: true, @@ -548,7 +433,7 @@ mod tests { use asap_executor::summary_kernels::HllSketchAccumulator; use asap_executor::values::Value; use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::ir::schema::{ + use asap_types::summary_type::{ NonNegativeWeightProof, SummaryInputExpr, SummaryUpdate, WeightDomain, }; use std::mem::{align_of, size_of}; @@ -589,10 +474,7 @@ mod tests { let families = [ ( A::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, + SketchParams::Kll { k: 200 }, Rows::Values, Priced::Kll { k: 200 }, 0, @@ -654,10 +536,7 @@ mod tests { ), ( A::Hll, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, + SketchParams::Hll { precision: 12 }, Rows::Values, Priced::Exact, hll_precision_tag, @@ -665,10 +544,7 @@ mod tests { ), ( A::Kmv, - SketchParams::Kmv { - k: 1_024, - relative_error: None, - }, + SketchParams::Kmv { k: 1_024 }, Rows::Values, Priced::Exact, 0, @@ -683,7 +559,6 @@ mod tests { sketch_rows: 1, sketch_cols: 16, layers: 4, - distinct_failure_probability: None, }, Rows::Labels, Priced::Bound, @@ -727,18 +602,13 @@ mod tests { assert_eq!(key.to_bytes().unwrap().len(), ASSUMED_HEAP_KEY_BYTES); for (algorithm, params, rows, priced_as, fields, full) in &families { for n in [1, 100, *full] { - let family = FieldDataType::Sketch( - SketchKind::new(*algorithm, params.clone()), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(*algorithm); let input = match rows { Rows::Values => &value, Rows::Labels => &labeled, Rows::Keyed => &keyed, }; - let mut updater = - create_planner_accumulator(&family, input, &GroupingStrategy::default()) - .unwrap(); + let mut updater = create_planner_accumulator(&family, Some(params), input).unwrap(); for i in 0..n { match rows { // Distinct magnitudes over the assumed range, in a @@ -802,34 +672,4 @@ mod tests { } } } - - /// A UnivMon row updates layers `0..=b`, where its bottom layer `b` is - /// geometric (`asap_sketchlib`'s `bottom_layer_for_hash`), so - /// `2 - 2^(1 - layers)` layers in expectation; each updates one counter - /// per sketch row and its heap. - #[test] - fn univmon_update_cost_is_expected_layers_times_rows_and_heap() { - let ops = |layers| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 256, - sketch_rows: 5, - sketch_cols: 1_024, - layers, - distinct_failure_probability: None, - }, - ), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .0 - }; - assert_eq!(ops(1), 6.0); - assert_eq!(ops(2), 1.5 * 6.0); - assert_eq!(ops(16), (2.0 - 2f64.powi(-15)) * 6.0); - } } diff --git a/crates/cost/src/node/usage.rs b/crates/cost/src/node/usage.rs new file mode 100644 index 000000000..ef3286ac3 --- /dev/null +++ b/crates/cost/src/node/usage.rs @@ -0,0 +1,863 @@ +//! One node of a physical plan DAG: its resource usage ([`node_usage`]) +//! and cost ([`NodeCost`]). + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use crate::node::analytical::{ + estimate_operator, AnalyticalCostError, PhysicalOperator, ResourceEstimate, +}; +use crate::node::operator_statistics::{ + EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics, +}; +use crate::node::state_bytes::{ + keeps_every_value, row_bytes, sketch_shape, summary_shape, StateInputRows, +}; +use crate::tumbling_windows_of; +use crate::{aggregate_lookup, lookup, StateFamily}; +use crate::{ + CalibrationFit, CostInputs, CostModel, NodeCostSource, OperatorKind, COST_SOURCE, + MEASURED_COST_SOURCE, MIXED_COST_SOURCE, +}; +use asap_types::ir::operator::{GroupKeys, Reduction, Source}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::schema::{FieldDataType, Schema}; +use asap_types::ir::{ASAPOp, NonASAPOp, TimeRangeKind}; +use asap_types::plan::physical_asap_dag::{ + PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, +}; +use asap_types::stages::{Lifetime, StorageMedium}; +use asap_types::summary_type::{SketchParams, SketchStatistic}; +use asap_types::workload::bounds::{ + declared_non_negative, declared_series, group_count_bound, source_name, source_rows_per_ms, + unchanged_source_columns, +}; +use asap_types::workload::DataWorkload; + +/// One node's price per second of wall time, with how it was measured. A +/// node is one operator node of the candidate's physical plan DAG +/// (`PhysicalASAPDAG`), keyed by its [`PhysicalASAPNodeId`] in +/// [`PlanCost::per_node`](crate::PlanCost::per_node); not a machine. +#[derive(Debug, Clone, PartialEq)] +pub struct NodeCost { + /// Its operator, as the measured table names it. + pub operator: OperatorKind, + /// When it runs: ingestion or query time. + pub timing: ExecutionTiming, + /// Its price per second under cost model 1; `None` under cost models 2 + /// and 3, which price (or limit) the whole candidate's peaks. + pub cost_per_second: Option, + /// Core-seconds of CPU it takes per second of wall time: its CPU + /// operations and scanned bytes timed by [`ResourceSpeed`](crate::ResourceSpeed), what a + /// runtime measures it against. A query's fixed per-request cost + /// ([`MeasuredOperationCosts::ns_per_evaluation`](crate::MeasuredOperationCosts::ns_per_evaluation)) is counted on the + /// node the query reads first. + pub cpu_core_seconds_per_second: f64, + /// The number of consumers it is computed for, with the + /// consumers materialized at ingestion time of one refresh window counted once (1 + /// when materialized, [`execution_demand::computations`](crate::plan::execution_demand::computations)). + pub multiplier: u64, + /// How it was measured, e.g. "scan 1000 samples; x 1.0000/s; keeps + /// ... bytes": one computation's work, the times it is computed per + /// second and what it keeps. + pub detail: String, + /// Whether its CPU operations were costed from the measured table or + /// from the formulas. + pub cost_source: NodeCostSource, +} + +/// The state family of a node with `payload` whose first producer has +/// `producer_family`: a build's own (one sketch instance per group; a +/// shared Hydra structure is not that sketch), inherited by the merges, +/// estimates and finalizations over it, none otherwise. +fn state_family<'a>( + payload: &'a Payload, + producer_family: Option>, +) -> StateFamily<'a> { + match payload { + Payload::ASAP( + ASAPOp::SummaryAgg { + summary_type: family, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + summary_type: family, + .. + }, + ) => match family { + FieldDataType::Sketch(..) => { + todo!("#1207 PR 6: the state family of θ from the candidate's LogicalProperties::parameters") + } + FieldDataType::ExactAggregate(kind, _) => StateFamily::Exact(kind), + _ => StateFamily::None, + }, + Payload::ASAP( + ASAPOp::SummaryMerge { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. }, + ) => producer_family.unwrap_or(StateFamily::None), + _ => StateFamily::None, + } +} + +/// One producer of a node being priced ([`node_usage`]): its payload and +/// schema, its output statistics, the sources it reads and its state +/// family. +#[derive(Debug, Clone)] +pub struct ProducerOutput<'a> { + pub payload: &'a Payload, + pub output_schema: &'a Schema, + pub output: EdgeStatistics, + pub sources: BTreeSet<&'a str>, + pub family: StateFamily<'a>, + /// Its output columns that still hold one source row's label (or + /// column) values unchanged ([`unchanged_source_columns`]). + pub unchanged_columns: BTreeSet<&'a str>, +} + +/// What one node uses and outputs ([`node_usage`]): its output statistics, sources and state family for its +/// consumers; its work for one computation (one execution) in ms on one +/// core; its usage per second; what is charged +/// once the whole plan is known (memory held for a query's evaluation, an +/// in-memory ingestion window's partial copies, the raw data a query-time +/// scan keeps on disk); its calibration fit and its [`NodeCost`]. +#[derive(Debug, Clone)] +pub struct NodeUsage<'a> { + pub output: EdgeStatistics, + pub sources: BTreeSet<&'a str>, + pub family: StateFamily<'a>, + /// Its output columns that still hold one source row's label (or + /// column) values unchanged ([`unchanged_source_columns`]). + pub unchanged_columns: BTreeSet<&'a str>, + pub work_ms_per_execution: f64, + pub cpu_ops_per_second: f64, + pub scanned_bytes_per_second: f64, + pub average_memory_bytes: f64, + pub peak_memory_bytes: u64, + pub disk_bytes: u64, + /// Query-time memory held while an evaluation runs: (bytes, computations + /// per second, whether each consumer computes its own copy, held while + /// that consumer's evaluation runs ([`plan_cost`](crate::plan_cost)), rather than one copy + /// held until the longest query reading the node finishes). + pub held: Option<(u64, f64, bool)>, + /// An in-memory ingestion-time state's bytes per window. + pub maintained: Option, + /// A query-time scan's source and the raw bytes it reads. + pub raw_scan: Option<(&'a Source, u64)>, + pub fit: CalibrationFit, + pub cost: NodeCost, +} + +/// Measure and price one node of a physical DAG (one step of +/// [`plan_cost`](crate::plan_cost)): `producers` are its producers in edge order, +/// `rate` its `(multiplier, per_second)` ([`computations`](crate::plan::execution_demand::computations)), +/// `retained_tumbling_window_count` the tumbling windows a tumbling-window chain keeps +/// completed (`retained_tumbling_windows`), and `ingestion_window_ms` the window an +/// ingestion-time node maintains (`maintained_window_ms`). +#[allow(clippy::too_many_arguments)] +pub fn node_usage<'a>( + node: &'a asap_types::plan::physical_asap_dag::PhysicalASAPDAGNode, + producers: &[ProducerOutput<'a>], + rate: (u64, f64), + retained_tumbling_window_count: u64, + ingestion_window_ms: Option, + data: &DataWorkload, + settings: &'a CostInputs, + formula_scale: f64, +) -> Result, AnalyticalCostError> { + let table = &settings.measured_operation_costs; + let speed = settings.resource_speed; + let usage_prices = match &settings.cost_model { + CostModel::UsageOverTime(prices) => Some(prices), + _ => None, + }; + let sketch_size = + |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; + let mut fit = CalibrationFit::default(); + let mut held = None; + let mut maintained = None; + let mut raw_scan = None; + // A query-time scan's raw rows retained. + let mut raw_retained_rows = None; + let ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; + let inputs: Vec = producers.iter().map(|p| p.output).collect(); + let sources: BTreeSet<&str> = match &node.payload { + Payload::NonASAP(NonASAPOp::Scan { source, .. }) => [source_name(source)].into(), + _ => producers + .iter() + .flat_map(|p| p.sources.iter().copied()) + .collect(), + }; + // The series its sources declare, when they all do. + let series = declared_series(&sources, data); + // The rows per ms its sources receive, which its scans and time + // ranges read, when the data declares an arrival rate. + let rows_per_ms = source_rows_per_ms(&sources, data); + // The fact a rate needs, named when it is missing. + let missing_rate = |source: &str| AnalyticalCostError::MissingSourceFact { + source_name: source.to_string(), + missing: "an arrival rate (the arrival's rows_per_second, or the \ + source's series_count with sample_interval)", + }; + let family = state_family(&node.payload, producers.first().map(|p| p.family)); + let unchanged_columns = unchanged_source_columns( + node, + &producers + .iter() + .map(|p| (p.output_schema, &p.unchanged_columns)) + .collect::>(), + ); + let operator = OperatorKind::of(&node.payload); + let unit_price = match &node.payload { + Payload::NonASAP(NonASAPOp::Aggregate { measures, .. }) => { + aggregate_lookup(table, node.output_state.timing, measures) + } + _ => lookup( + table, + operator, + node.output_state.timing, + family, + sketch_size, + ), + }; + // The kernel's measured bytes per state, when the table has them. + let measured_state_bytes = unit_price + .and_then(|p| p.memory_bytes_per_state) + .map(|bytes| bytes.round() as u64); + let mut cost_source = NodeCostSource::Analytical; + let input = inputs + .first() + .copied() + .unwrap_or(EdgeStatistics { rows: 1, bytes: 1 }); + let width = row_bytes(&node.output_schema); + let edge = |rows: u64| -> Result { + Ok(EdgeStatistics { + rows, + bytes: rows + .checked_mul(width) + .ok_or(AnalyticalCostError::Overflow)?, + }) + }; + // Whether a summary built here sees raw data samples (its producer a + // scan or a time range) of sources all declared non-negative. + let raw_data_samples_non_negative = || { + let raw = producers.first().is_some_and(|producer| { + matches!( + producer.payload, + Payload::NonASAP(NonASAPOp::Scan { .. } | NonASAPOp::TimeRange { .. }) + ) + }); + raw && sources + .iter() + .all(|source| declared_non_negative(data, source)) + }; + let unary = |output| UnaryEdgeStatistics { + input, + output, + promql: None, + }; + // The schema the node's group and partition keys index. + let input_schema = producers.first().map(|p| p.output_schema); + // The rows a grouping reads per evaluation: its input at query + // time; at ingestion time, the arriving rows of the window it + // maintains (one second of them does not bound its groups). + // Query time always has its input rows; an ingestion-time grouping + // with no window below it is an error (Stage 2 builds none). + let rows_read = || match ingestion { + false => Ok(input.rows.max(1)), + true => { + let window_ms = + ingestion_window_ms.ok_or(AnalyticalCostError::IngestionGroupingWithoutWindow)?; + // Name a source with no rate of its own. + let rows_per_ms = rows_per_ms.ok_or_else(|| { + let unrated = sources + .iter() + .find(|source| source_rows_per_ms(&[**source].into(), data).is_none()); + missing_rate(unrated.copied().unwrap_or("(none)")) + })?; + Ok(((rows_per_ms * window_ms as f64).round() as u64).max(1)) + } + }; + let estimated_groups = |keys: &GroupKeys| { + let rows_read = rows_read()?; + Ok(group_count_bound( + keys, + input_schema, + &sources, + producers + .first() + .map_or(&BTreeSet::new(), |producer| &producer.unchanged_columns), + data, + rows_read, + )) + }; + let groups = |reduction: &Reduction| match reduction { + Reduction::Reduce(keys) if keys.keys().is_empty() && !keys.is_without() => Ok(1), + Reduction::Reduce(keys) => estimated_groups(keys), + // One group per series: the declared series (at query time at + // most the rows read), else the rows read, each series having at + // least one. + Reduction::PerEntity => match (ingestion, series) { + (true, Some(series)) => Ok(series.max(1)), + (false, Some(series)) => Ok(series.min(input.rows).max(1)), + (_, None) => rows_read(), + }, + }; + let (out, estimate, detail) = match &node.payload { + Payload::NonASAP(operator) => match operator { + NonASAPOp::Scan { source, .. } => { + // At ingestion time, one second of arriving rows; at + // query time, the arriving rows of the window read above + // it, at most the rows the source holds (all of them + // without a window). + let source = source_name(source); + let rows_held = data.sources.get(source).and_then(|facts| facts.rows_held); + let (window_width_ms, retained_lookback_ms): (Option, Option) = + match ingestion { + true => (None, None), + false => todo!("#1207 PR 7: a query-time Scan's read per edge (the TimeRange above it) and its retained rows from retention_ms (h_v)"), + }; + let lookback = match ingestion { + true => Some(1_000), + false => window_width_ms, + }; + let by_rate = |lookback: Option| { + rows_per_ms.zip(lookback).map(|(rows_per_ms, lookback)| { + (rows_per_ms * lookback as f64).round() as u64 + }) + }; + // The raw rows retained for a query-time scan: back to the + // oldest row it reads, `end_offset + width` (an offset + // window keeps more than it reads), at most those held. + if !ingestion { + let retained_rows = match (by_rate(retained_lookback_ms), rows_held) { + (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), + (by_rate, rows_held) => by_rate.or(rows_held), + }; + raw_retained_rows = retained_rows.map(|rows| rows.max(1)); + } + let by_rate = by_rate(lookback); + let rows = match (ingestion, by_rate, rows_held) { + (true, by_rate, _) => by_rate.ok_or_else(|| missing_rate(source))?, + (false, Some(by_rate), Some(rows_held)) => by_rate.min(rows_held), + (false, by_rate, rows_held) => by_rate.or(rows_held).ok_or_else(|| { + AnalyticalCostError::MissingSourceFact { + source_name: source.to_string(), + missing: match lookback { + Some(_) => { + "an arrival rate (the arrival's rows_per_second, or the \ + source's series_count with sample_interval) or its \ + rows_held" + } + None => "its rows_held (the scan has no window, so it reads every row held)", + }, + } + })?, + }; + let out = edge(rows.max(1))?; + let estimate = estimate_operator( + PhysicalOperator::Scan, + OperatorStatistics::Scan { + edges: UnaryEdgeStatistics { + input: out, + output: out, + promql: None, + }, + source_read_bytes: out.bytes, + }, + ); + (out, estimate, format!("scan {} samples", out.rows)) + } + NonASAPOp::Aggregate { + reduction, + measures, + .. + } => { + let group_count = groups(reduction)?; + let keys = match reduction { + Reduction::Reduce(keys) => keys.keys().len() as u64, + _ => 1, + }; + let out = edge(group_count)?; + let held = input.rows.div_ceil(group_count.max(1)); + let estimate = estimate_operator( + PhysicalOperator::HashAggregate { + grouping_key_count: keys, + accumulator_count: measures.len().max(1) as u64, + }, + OperatorStatistics::HashAggregate { + edges: unary(out), + group_count, + key_bytes: keys.checked_mul(16).ok_or(AnalyticalCostError::Overflow)?, + accumulator_bytes_per_group: measures + .iter() + .try_fold(0_u64, |total, measure| { + let values = if keeps_every_value(measure) { held } else { 1 }; + total + .checked_add( + values + .checked_mul(8) + .ok_or(AnalyticalCostError::Overflow)?, + ) + .ok_or(AnalyticalCostError::Overflow) + })? + .max(8), + }, + ); + ( + out, + estimate, + format!( + "hash aggregate {} rows into {group_count} groups", + input.rows + ), + ) + } + NonASAPOp::Sort { + keys, partition_by, .. + } => { + let partitions = if partition_by.keys().is_empty() { + 1 + } else { + estimated_groups(partition_by)? + }; + let estimate = estimate_operator( + PhysicalOperator::InMemoryComparisonSort { + ordering_key_count: keys.len() as u64, + partitioned: partitions > 1, + }, + OperatorStatistics::InMemoryComparisonSort { + edges: unary(input), + input_partitioning: split(input, partitions), + }, + ); + ( + input, + estimate, + format!("sort {} rows in {partitions} partitions", input.rows), + ) + } + NonASAPOp::Limit { + n, + offset, + partition_by, + .. + } => { + let partitions = if partition_by.keys().is_empty() { + 1 + } else { + estimated_groups(partition_by)? + }; + let limit = n.map_or(u64::MAX, |n| (n as u64).saturating_mul(partitions)); + let offset = (*offset as u64).saturating_mul(partitions); + let out = edge(input.rows.saturating_sub(offset).min(limit))?; + let estimate = estimate_operator( + PhysicalOperator::Limit { limit, offset }, + OperatorStatistics::Limit { edges: unary(out) }, + ); + (out, estimate, format!("limit to {} rows", out.rows)) + } + NonASAPOp::TimeShift { .. } => ( + input, + estimate_operator( + PhysicalOperator::TimeShift, + OperatorStatistics::PassThrough { + edges: unary(input), + }, + ), + format!("time shift: pass {} rows, free", input.rows), + ), + NonASAPOp::TimeRange { range, kind, .. } => { + // A range keeps its window's samples; an instant + // selector the latest sample of each series. + // Unknown rate or series: no fewer than its input. + let retained = match kind { + TimeRangeKind::Range => rows_per_ms.map_or(input.rows, |rows_per_ms| { + (rows_per_ms * range.as_millis() as f64).round() as u64 + }), + TimeRangeKind::Instant => series.unwrap_or(input.rows), + }; + let out = edge(input.rows.min(retained.max(1)))?; + let out = EdgeStatistics { + rows: out.rows, + bytes: out.bytes.min(input.bytes), + }; + let estimate = estimate_operator( + PhysicalOperator::TimeRange, + OperatorStatistics::PassThrough { edges: unary(out) }, + ); + ( + out, + estimate, + format!( + "time range {range:?}: keep {} of {} rows", + out.rows, input.rows + ), + ) + } + _ => { + let estimate = estimate_operator( + PhysicalOperator::PassThrough, + OperatorStatistics::PassThrough { + edges: unary(input), + }, + ); + ( + input, + estimate, + format!("operator: pass {} rows", input.rows), + ) + } + }, + Payload::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + reduction, + .. + }) => { + let group_count = groups(reduction)?; + // At query time, each state sees its group's input rows; at + // ingestion time, the rows of a window one second of + // arriving rows does not bound. + let input_rows = StateInputRows { + rows_per_state: (!ingestion).then(|| input.rows.div_ceil(group_count.max(1))), + non_negative: raw_data_samples_non_negative(), + }; + let (depth, state_bytes) = summary_shape(family, input_rows); + let state_bytes = measured_state_bytes.unwrap_or(state_bytes); + let out = EdgeStatistics { + rows: group_count, + bytes: group_count + .checked_mul(state_bytes) + .ok_or(AnalyticalCostError::Overflow)?, + }; + ( + out, + Ok(ResourceEstimate::new( + input.rows as f64 * depth, + out.bytes, + 0, + )), + format!( + "build {} into {group_count} states: {} rows x depth {depth}", + summary_type_name(family), + input.rows + ), + ) + } + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { + todo!("#1207 PR 7: a chain's states and memory over the windows it retains (h_v)") + } + Payload::ASAP(ASAPOp::SummaryMerge { group_by, .. }) => { + // One operation per input state read; one state per group + // out. A rollup's groups are those of its keys, at most its + // input groups. + let mut group_count = 0; + for producer in producers { + if matches!( + producer.payload, + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) { + todo!("#1207 PR 7: the states a merge reads from a chain's windows") + } + group_count = group_count.max(producer.output.rows); + } + let states = merged_states(producers)?; + if let Some(keys) = group_by { + group_count = group_count.min(groups(&Reduction::by(keys.clone()))?); + } + let out = EdgeStatistics { + rows: group_count, + bytes: input + .bytes + .div_ceil(input.rows.max(1)) + .checked_mul(group_count) + .ok_or(AnalyticalCostError::Overflow)?, + }; + ( + out, + Ok(ResourceEstimate::new(states as f64, out.bytes, 0)), + format!("merge {states} states into {group_count}"), + ) + } + Payload::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { + let per_group = match query { + SketchStatistic::TopK { k } => *k as u64, + _ => 1, + }; + let out = edge( + input + .rows + .checked_mul(per_group) + .ok_or(AnalyticalCostError::Overflow)?, + )?; + ( + out, + Ok(ResourceEstimate::new(out.rows as f64, 0, 0)), + format!("estimate {} groups x {per_group}", input.rows), + ) + } + Payload::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) => ( + edge(input.rows)?, + Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), + format!("finalize {} accumulators", input.rows), + ), + _ => ( + edge(input.rows)?, + Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), + format!("{} rows", input.rows), + ), + }; + let estimate = estimate?; + let (multiplier, per_second) = rate; + let formula_ms = estimate.cpu_ops() * speed.ms_per_cpu_op + + estimate.scan_bytes() as f64 * speed.ms_per_scanned_byte; + // What one computation processes, the unit its entry is priced per + // ([`OperatorKind`]): the samples a scan reads, the states a merge + // reads, otherwise the rows of its (first) input. + let units = match &node.payload { + Payload::NonASAP(NonASAPOp::Scan { .. }) => out.rows, + Payload::ASAP(ASAPOp::SummaryMerge { .. }) => merged_states(producers)?, + _ => input.rows, + }; + let mut detail = detail; + // A node with an entry takes its measured time, scanning included; + // any other node its formulas' CPU operations times the calibration + // factor, its scanned bytes unscaled (scaling only the CPU keeps it + // non-negative whatever the factor). + let estimate = match unit_price { + Some(unit) => { + cost_source = NodeCostSource::Measured; + let measured_ms = (units as f64 * unit.ns_per_unit + unit.ns_per_invocation) * 1e-6; + fit.record(measured_ms * per_second, formula_ms * per_second); + detail += &format!( + "; measured {units} x {:.3} ns + {:.3} ns", + unit.ns_per_unit, unit.ns_per_invocation + ); + ResourceEstimate::new( + measured_ms / speed.ms_per_cpu_op, + estimate.peak_memory_bytes(), + 0, + ) + } + None if formula_scale != 1.0 => { + detail += &format!("; formula x {formula_scale:.3} (calibration)"); + ResourceEstimate::new( + estimate.cpu_ops() * formula_scale, + estimate.peak_memory_bytes(), + estimate.scan_bytes(), + ) + } + None => estimate, + }; + let node_ms = estimate.cpu_ops() * speed.ms_per_cpu_op + + estimate.scan_bytes() as f64 * speed.ms_per_scanned_byte; + + // What this node keeps: memory averaged over time (byte-seconds per + // second), its peak memory and its disk bytes, added to the + // candidate's usage below. + let mut average_memory_bytes = 0.0; + let mut peak_memory_bytes = 0u64; + let mut disk_bytes = 0u64; + let mut detail = format!("{detail}; x {per_second:.4}/s"); + // State retained between computations: at ingestion time, the window + // being built and the completed ones queries read (a chain of N + // tumbling windows keeps N + 1), every second. At query time the output is + // reused only within one evaluation, so it is held only while that + // evaluation runs (counted below, once latencies are known). The + // state is what the node holds: its output, or the values an exact + // aggregate keeps per group when that is more. A computation that + // keeps nothing holds its working memory while it runs, once per + // consumer that recomputes it; at query time, that is held for the + // evaluation too (counted in [`plan_cost`](crate::plan_cost), per consumer: a query's + // latency, or a materialized consumer's longest query's, as a + // OneEvaluation state is). + let state = out.bytes.max(estimate.peak_memory_bytes()); + let (materialized_bytes, medium) = match (ingestion, node.lifetime, &node.payload) { + (_, Lifetime::None, _) => { + let working_memory_bytes = estimate.peak_memory_bytes(); + peak_memory_bytes = working_memory_bytes.saturating_mul(multiplier); + if !ingestion && working_memory_bytes > 0 { + held = Some((working_memory_bytes, per_second, true)); + } + (0, StorageMedium::Memory) + } + // A tumbling-window chain materialized across evaluations holds the tumbling windows its + // retention covers ([`tumbling_window_retention_ms`]: N, plus one per tumbling window + // width of the queries' range-evaluation span), all but the + // newest between evaluations and all while one runs. + ( + false, + Lifetime::AcrossEvaluations(medium), + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), + ) => { + let tumbling_windows = tumbling_windows_of(node); + let retained = retained_tumbling_window_count; + let between = (state / tumbling_windows) + .checked_mul( + retained + .checked_sub(1) + .ok_or(AnalyticalCostError::Overflow)?, + ) + .ok_or(AnalyticalCostError::Overflow)?; + match medium { + StorageMedium::Memory => { + average_memory_bytes = between as f64; + peak_memory_bytes = match retained == tumbling_windows { + true => state, + false => (state / tumbling_windows) + .checked_mul(retained) + .ok_or(AnalyticalCostError::Overflow)?, + }; + } + StorageMedium::Disk => disk_bytes = between, + } + detail += &format!("; materializes {between} bytes across evaluations"); + (0, medium) + } + (_, Lifetime::OneEvaluation(medium), _) => { + match medium { + StorageMedium::Memory => { + held = Some((state, per_second, false)); + peak_memory_bytes = state; + } + StorageMedium::Disk => disk_bytes = state, + } + (0, medium) + } + // The retained tumbling windows and the one being built. + ( + true, + Lifetime::AcrossEvaluations(medium), + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), + ) => { + let tumbling_windows = tumbling_windows_of(node); + ( + (state / tumbling_windows) + .checked_mul( + retained_tumbling_window_count + .checked_add(1) + .ok_or(AnalyticalCostError::Overflow)?, + ) + .ok_or(AnalyticalCostError::Overflow)?, + medium, + ) + } + // A whole state materialized across evaluations: it and the one being built. + (_, Lifetime::AcrossEvaluations(medium), _) => ( + state.checked_mul(2).ok_or(AnalyticalCostError::Overflow)?, + medium, + ), + }; + if ingestion && node.lifetime == Lifetime::AcrossEvaluations(StorageMedium::Memory) { + maintained = Some(state / tumbling_windows_of(node)); + } + if materialized_bytes > 0 { + match medium { + StorageMedium::Memory => { + average_memory_bytes = materialized_bytes as f64; + peak_memory_bytes = materialized_bytes; + } + StorageMedium::Disk => disk_bytes = materialized_bytes, + } + detail += &format!("; keeps {materialized_bytes} bytes"); + } + if let Payload::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload { + if !ingestion { + let retained = match raw_retained_rows { + Some(rows) => edge(rows)?.bytes, + None => out.bytes, + }; + raw_scan = Some((source, retained)); + } + } + // One computation's CPU and scan price times its rate, then the + // materialized state's. + let cost = usage_prices.map(|prices| { + (estimate.cpu_ops() * prices.cost_per_cpu_op + + estimate.scan_bytes() as f64 * prices.cost_per_scanned_byte) + * per_second + + prices.cost_per_memory_byte_second * average_memory_bytes + + prices.cost_per_disk_byte_second * disk_bytes as f64 + }); + if cost.is_some_and(|cost| !cost.is_finite()) { + return Err(AnalyticalCostError::Overflow); + } + Ok(NodeUsage { + output: out, + sources, + family, + unchanged_columns, + work_ms_per_execution: node_ms, + cpu_ops_per_second: estimate.cpu_ops() * per_second, + scanned_bytes_per_second: estimate.scan_bytes() as f64 * per_second, + average_memory_bytes, + peak_memory_bytes, + disk_bytes, + held, + maintained, + raw_scan, + fit, + cost: NodeCost { + operator, + timing: node.output_state.timing, + cost_per_second: cost, + cpu_core_seconds_per_second: node_ms * per_second / 1_000.0, + multiplier, + detail, + cost_source, + }, + }) +} + +/// [`PlanCost::source`] for nodes priced as `per_node` says. +pub(crate) fn cost_source(per_node: &BTreeMap) -> &'static str { + let measured = |node: &NodeCost| node.cost_source == NodeCostSource::Measured; + if !per_node.values().any(measured) { + COST_SOURCE + } else if per_node.values().all(measured) { + MEASURED_COST_SOURCE + } else { + MIXED_COST_SOURCE + } +} + +/// The input states a `SummaryMerge` over `producers` reads: every row of +/// each. +fn merged_states(producers: &[ProducerOutput<'_>]) -> Result { + producers.iter().try_fold(0_u64, |total, producer| { + total + .checked_add(producer.output.rows) + .ok_or(AnalyticalCostError::Overflow) + }) +} + +/// The nodes of `dag` by id. +pub fn node_map(dag: &PhysicalASAPDAG) -> NodeMap<'_> { + dag.nodes.iter().map(|n| (n.id, n)).collect() +} + +/// The nodes of a physical DAG by id. +pub type NodeMap<'a> = + HashMap; + +/// A summary type's name in reports: a sketch by its algorithm, e.g. +/// `HydraCms`; an exact accumulator by its kind. +pub fn summary_type_name(summary_type: &FieldDataType) -> String { + match summary_type { + FieldDataType::Sketch(algorithm) => format!("{algorithm:?}"), + FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), + other => format!("{other:?}"), + } +} + +/// `input` split as evenly as integers allow into `partitions` parts. +fn split(input: EdgeStatistics, partitions: u64) -> PartitionStatistics { + let part = |total: u64, i: u64| total / partitions + u64::from(i < total % partitions); + PartitionStatistics { + partitions: (0..partitions) + .map(|i| EdgeStatistics { + rows: part(input.rows, i), + bytes: part(input.bytes, i), + }) + .collect(), + } +} diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs new file mode 100644 index 000000000..c9e56a904 --- /dev/null +++ b/crates/cost/src/plan/cost.rs @@ -0,0 +1,573 @@ +//! Costing a physical plan DAG node by node: each node's resource usage +//! ([`node_usage`]), then the plan's usage, latency, retention, retained +//! tumbling window chains and cost ([`plan_cost`]). Moved from Stage 3 +//! (`asap-plan-selection`, whose module doc, "Cost model", describes the +//! model), so Stage 3 computes no cost of its own. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use crate::node::analytical::AnalyticalCostError; +use crate::node::operator_statistics::EdgeStatistics; +use crate::node::state_bytes::{sketch_shape, StateInputRows}; +use crate::node::usage::cost_source; +use crate::plan::execution_demand::{ + computations, consumers, evaluation_rate, query_starts, Consumer, +}; +use crate::{lookup, StateFamily}; +use crate::{ + maintained_window_ms, node_usage, query_reach, retained_tumbling_windows, + tumbling_window_retention_ms, tumbling_windows_of, NodeMap, NodeUsage, ProducerOutput, +}; +use crate::{ + CalibrationFit, CostInputs, CostModel, NodeCost, NodeCostSource, OperatorKind, ResourceUsage, + MEASURED_COST_SOURCE, MIXED_COST_SOURCE, +}; +use asap_types::ir::operator::Source; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::summary_type::SketchParams; +use asap_types::workload::{DataWorkload, QueryRecurrence, RangeEvaluation}; + +/// One candidate's usage and price over the whole workload: `usage` is +/// the whole candidate's, `per_node` one price per operator node of its +/// physical plan DAG. +#[derive(Debug, Clone, PartialEq)] +pub struct PlanCost { + pub usage: ResourceUsage, + /// Its price per second under cost model 1 or 2 (under cost model 1, the + /// sum of its nodes' charges plus request overhead for queries without + /// operator nodes); `None` under cost model 3. + pub cost_per_second: Option, + /// Where its prices came from, from its nodes' [`NodeCost::cost_source`]: + /// [`COST_SOURCE`](crate::COST_SOURCE) (all formulas), [`MEASURED_COST_SOURCE`] (all + /// measured) or [`MIXED_COST_SOURCE`]. + pub source: &'static str, + /// Each operator node of the candidate's physical plan DAG, by its id + /// in that DAG, with its price. + pub per_node: BTreeMap, +} + +/// One query's latency per evaluation, its longest chain of work (module +/// doc, "Latency"): the compaction of the newest window of a state it reads +/// from ingestion time, then the query's own work, each on a full core. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct QueryLatency { + /// Milliseconds to merge the ingest workers' partial copies of the + /// newest window of the largest ingestion-time state the query reads; + /// 0 when ingestion fits on one core or the query reads none. + pub compaction_ms: f64, + /// Milliseconds of the query-time work the query reaches, each node + /// once. + pub query_ms: f64, +} + +impl QueryLatency { + /// `compaction_ms + query_ms`: what is checked against the query's + /// `max_latency_ms`. + pub fn total_ms(&self) -> f64 { + self.compaction_ms + self.query_ms + } +} + +/// Measure every node of `dag` per second of wall time (module doc, "Cost +/// model") and price it under `settings.cost_model`. Nodes are exported +/// children first, so each node's input statistics are known when it is +/// reached. A query-time node is measured per computation over the rows its +/// time ranges keep, times how often it is computed per second +/// ([`computations`]); an ingestion-time node over one second of arriving +/// rows. A materialized node's state is counted in byte-seconds per +/// second; a tumbling-window chain materialized beyond one evaluation keeps the tumbling windows its +/// retention covers ([`tumbling_window_retention_ms`]). `nodes` is `dag`'s +/// ([`node_map`](crate::node_map)); `recurrence[q]` and +/// `range_evaluations[q]` say how often query `q` runs and how many steps +/// each run evaluates (one when missing). +#[allow(clippy::too_many_arguments)] +pub fn plan_cost( + dag: &PhysicalASAPDAG, + nodes: &NodeMap<'_>, + recurrence: &[QueryRecurrence], + range_evaluations: &[Option], + data: &DataWorkload, + settings: &CostInputs, + formula_scale: f64, +) -> Result<(PlanCost, CalibrationFit), (PhysicalASAPNodeId, AnalyticalCostError)> { + let first = dag.nodes.first().map(|n| n.id).unwrap_or_default(); + let table = &settings.measured_operation_costs; + let mut fit = CalibrationFit::default(); + let speed = settings.resource_speed; + // Cost model 1 prices each node as it is measured. + let usage_prices = match &settings.cost_model { + CostModel::UsageOverTime(prices) => Some(prices), + _ => None, + }; + let EachNodeUsage { + priced, + multipliers, + reach, + } = each_node_usage( + dag, + nodes, + recurrence, + range_evaluations, + data, + settings, + formula_scale, + None, + )?; + let mut output: HashMap = HashMap::new(); + let mut per_node = BTreeMap::new(); + // The candidate's usage, accumulated node by node; peak cores and + // latency are filled in once every node is measured. + let mut usage = ResourceUsage::default(); + // Each node's work in ms on one core: one computation at query time, + // one second of arriving rows at ingestion time. + let mut work_ms: HashMap = HashMap::new(); + // Query-time memory held only while one evaluation runs, a OneEvaluation + // state in memory or a not-materialized node's working memory: (bytes, + // per second, whether each consumer computes its own copy), + // counted once the evaluations' latencies are known, in node order so + // the sum is the same in every run. + let mut held: BTreeMap = BTreeMap::new(); + // The raw data query-time scans read, retained on disk: per source, the + // largest scan's (bytes, node). + let mut raw_data: HashMap<&Source, (u64, PhysicalASAPNodeId)> = HashMap::new(); + // Each in-memory ingestion-time state's bytes per window (what it keeps + // for one window, its output or an exact aggregate's values). + let mut maintained: Vec<(PhysicalASAPNodeId, u64)> = Vec::new(); + // Each summary node's state family, to look up in the measured table: + // a build's own (one sketch instance per group; a shared Hydra + // structure is not that sketch), inherited by the merges, estimates and + // finalizations over it. + let mut family_of: HashMap> = HashMap::new(); + let sketch_size = + |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; + for node in &dag.nodes { + let node_priced = &priced[&node.id]; + usage.cpu_ops_per_second += node_priced.cpu_ops_per_second; + usage.scanned_bytes_per_second += node_priced.scanned_bytes_per_second; + usage.average_memory_bytes += node_priced.average_memory_bytes; + usage.peak_memory_bytes = usage + .peak_memory_bytes + .saturating_add(node_priced.peak_memory_bytes); + usage.disk_bytes = usage.disk_bytes.saturating_add(node_priced.disk_bytes); + fit.add(node_priced.fit); + work_ms.insert(node.id, node_priced.work_ms_per_execution); + output.insert(node.id, node_priced.output); + if node_priced.family != StateFamily::None { + family_of.insert(node.id, node_priced.family); + } + if let Some(entry) = node_priced.held { + held.insert(node.id, entry); + } + if let Some(window) = node_priced.maintained { + maintained.push((node.id, window)); + } + if let Some((source, bytes)) = node_priced.raw_scan { + let largest = raw_data.entry(source).or_insert((bytes, node.id)); + if bytes > largest.0 { + *largest = (bytes, node.id); + } + } + per_node.insert(node.id, node_priced.cost.clone()); + } + // The runtime's fixed cost per query request, charged to the node the + // query reads first and added to its latency below. + let evaluation_ms = table.ns_per_evaluation * 1e-6; + let starts = query_starts(dag); + let mut evaluation_ms_per_query = vec![0.0; starts.len()]; + let mut request_ms_per_second = 0.0; + // Literal-only roots have no operator to own their fee. Keep that charge + // at plan level while preserving existing attribution for operator queries. + let mut unattached_request_cost = 0.0; + let mut has_unattached_request_overhead = false; + for (query, start) in starts.iter().enumerate() { + if evaluation_ms <= 0.0 { + continue; + } + let diagnostic_node = start.first().copied().unwrap_or(first); + let requests_per_second = evaluation_rate( + recurrence.get(query).map(|r| (r, None)), + settings.non_repeating_query_horizon_seconds, + ) + .map_err(|error| (diagnostic_node, error))?; + let ops_per_second = evaluation_ms / speed.ms_per_cpu_op * requests_per_second; + usage.cpu_ops_per_second += ops_per_second; + request_ms_per_second += evaluation_ms * requests_per_second; + if !usage.cpu_ops_per_second.is_finite() || !request_ms_per_second.is_finite() { + return Err((diagnostic_node, AnalyticalCostError::Overflow)); + } + evaluation_ms_per_query[query] = evaluation_ms; + let Some(&first_read) = start.first() else { + has_unattached_request_overhead = true; + if let Some(prices) = usage_prices { + unattached_request_cost += prices.cost_per_cpu_op * ops_per_second; + if !unattached_request_cost.is_finite() { + return Err((diagnostic_node, AnalyticalCostError::Overflow)); + } + } + continue; + }; + let node = per_node + .get_mut(&first_read) + .expect("every node is measured"); + node.cpu_core_seconds_per_second += evaluation_ms * requests_per_second / 1_000.0; + if !node.cpu_core_seconds_per_second.is_finite() { + return Err((first_read, AnalyticalCostError::Overflow)); + } + node.detail += &format!( + "; q{} request overhead {:.3} ns x {requests_per_second:.4}/s", + query + 1, + table.ns_per_evaluation + ); + if let (Some(cost), Some(prices)) = (&mut node.cost_per_second, usage_prices) { + *cost += prices.cost_per_cpu_op * ops_per_second; + if !cost.is_finite() { + return Err((first_read, AnalyticalCostError::Overflow)); + } + } + } + for (bytes, id) in raw_data.into_values() { + let node = per_node.get_mut(&id).expect("every node is measured"); + usage.disk_bytes = usage.disk_bytes.saturating_add(bytes); + node.detail += &format!("; keeps {bytes} bytes of raw data on disk"); + if let (Some(cost), Some(prices)) = (&mut node.cost_per_second, usage_prices) { + *cost += prices.cost_per_disk_byte_second * bytes as f64; + } + } + // With k ingest workers, each of the k − 1 beyond the first builds its + // own partial copy of an in-memory ingestion-time state's newest window + // (merged at query time, [`query_latencies`]), held at the peak too. + for (id, window) in maintained { + let copies = ingest_workers(dag, &work_ms, id) - 1; + if copies == 0 { + continue; + } + let bytes = copies.saturating_mul(window); + let measured = per_node.get_mut(&id).expect("every node is measured"); + usage.peak_memory_bytes = usage.peak_memory_bytes.saturating_add(bytes); + measured.detail += + &format!("; {copies} partial copies of its newest window, {bytes} bytes"); + } + // Milliseconds of one merge of each state with a measured query-time + // merge entry (measured ns · 10⁻⁶), for the newest window's compaction. + let merge_ms: HashMap = family_of + .iter() + .filter_map(|(id, family)| { + lookup( + table, + OperatorKind::SummaryMerge, + ExecutionTiming::QueryTime, + *family, + sketch_size, + ) + .map(|unit| (*id, (unit.ns_per_unit + unit.ns_per_invocation) * 1e-6)) + }) + .collect(); + let mut latency = query_latencies( + dag, + nodes, + &output, + &work_ms, + &merge_ms, + speed.ms_per_cpu_op * formula_scale, + ); + for (latency, evaluation_ms) in latency.iter_mut().zip(evaluation_ms_per_query) { + latency.query_ms += evaluation_ms; + } + // Query-time memory is held for one evaluation: a `OneEvaluation` state from + // its build until the longest query reading it finishes, about that + // query's latency, once per execution. A not-materialized node's working + // memory is computed once per consumer ([`consumers`]), and each copy is + // held while its consumer's evaluation runs: a query's for that query's + // latency, at its rate; a materialized node's until the longest query + // reading that node finishes, at the node's rate, as its `OneEvaluation` + // state is. + let longest_reader_ms = |id: PhysicalASAPNodeId| { + reach + .iter() + .zip(&latency) + .filter(|(reached, _)| reached.contains(&id)) + .map(|(_, l)| l.total_ms()) + .fold(0.0, f64::max) + }; + let consumers_of = consumers(dag); + for (id, (state, per_second, computed_per_consumer)) in held { + // The milliseconds it is held per second: each execution's held + // time times its executions per second, summed. + let held_ms_per_second = match computed_per_consumer { + false => per_second * longest_reader_ms(id), + true => { + let mut sum = 0.0; + for &consumer in consumers_of.get(&id).into_iter().flatten() { + sum += match consumer { + Consumer::Query(query) => { + let rate = evaluation_rate( + recurrence + .get(query) + .map(|r| (r, range_evaluations.get(query).copied().flatten())), + settings.non_repeating_query_horizon_seconds, + ) + .map_err(|error| (id, error))?; + rate * latency.get(query).map_or(0.0, QueryLatency::total_ms) + } + Consumer::Materialized(node) => { + multipliers[&node].1 * longest_reader_ms(node) + } + }; + } + sum + } + }; + let evaluation_ms = match per_second > 0.0 { + true => held_ms_per_second / per_second, + false => 0.0, + }; + let byte_seconds = state as f64 * held_ms_per_second / 1_000.0; + let node = per_node.get_mut(&id).expect("every node is measured"); + usage.average_memory_bytes += byte_seconds; + node.detail += &format!("; holds {state} bytes for {evaluation_ms:.1} ms"); + if let (Some(cost), Some(prices)) = (&mut node.cost_per_second, usage_prices) { + *cost += prices.cost_per_memory_byte_second * byte_seconds; + if !cost.is_finite() { + return Err((id, AnalyticalCostError::Overflow)); + } + } + } + usage.peak_cpu_cores = peak_cpu_cores( + dag, + nodes, + &work_ms, + &multipliers, + &latency, + request_ms_per_second, + ); + usage.latency_per_query = latency; + let cost = match &settings.cost_model { + CostModel::UsageOverTime(_) => { + let node_cost: Option = per_node.values().map(|node| node.cost_per_second).sum(); + let total = node_cost.map(|cost| cost + unattached_request_cost); + if total.is_some_and(|cost| !cost.is_finite()) { + return Err((first, AnalyticalCostError::Overflow)); + } + total + } + CostModel::ProvisionedCapacity(prices) => { + let cost = prices.cost_per_provisioned_memory_byte_second + * usage.peak_memory_bytes as f64 + + prices.cost_per_provisioned_cpu_core_second * f64::from(usage.peak_cpu_cores) + + prices.cost_per_disk_byte_second * usage.disk_bytes as f64; + if !cost.is_finite() { + return Err((first, AnalyticalCostError::Overflow)); + } + Some(cost) + } + CostModel::ClusterCapacity(_) => None, + }; + Ok(( + PlanCost { + usage, + cost_per_second: cost, + source: if has_unattached_request_overhead { + if per_node + .values() + .all(|node| node.cost_source == NodeCostSource::Measured) + { + MEASURED_COST_SOURCE + } else { + MIXED_COST_SOURCE + } + } else { + cost_source(&per_node) + }, + per_node, + }, + fit, + )) +} + +/// Every node of a physical DAG priced ([`each_node_usage`]), with each +/// node's `(multiplier, per_second)` ([`computations`]) and the nodes each +/// query reaches at query time ([`query_reach`]). +struct EachNodeUsage<'a> { + priced: HashMap>, + multipliers: HashMap, + reach: Vec>, +} + +/// [`node_usage`] over every node of `dag`, children first, with the +/// arguments of [`plan_cost`]. `root_rates`, when given, are +/// the computations per second of each root's consumer in place of its +/// query's own rate ([`computations`]). +#[allow(clippy::too_many_arguments)] +fn each_node_usage<'a>( + dag: &'a PhysicalASAPDAG, + nodes: &NodeMap<'a>, + recurrence: &[QueryRecurrence], + range_evaluations: &[Option], + data: &DataWorkload, + settings: &'a CostInputs, + formula_scale: f64, + root_rates: Option<&[f64]>, +) -> Result, (PhysicalASAPNodeId, AnalyticalCostError)> { + let first = dag.nodes.first().map(|n| n.id).unwrap_or_default(); + let multipliers = computations( + dag, + recurrence, + range_evaluations, + settings.non_repeating_query_horizon_seconds, + root_rates, + ) + .map_err(|error| (first, error))?; + let reach = query_reach(dag, nodes); + let retention_ms = tumbling_window_retention_ms(dag, &reach, range_evaluations); + let mut producers_of: HashMap> = HashMap::new(); + for e in &dag.edges { + producers_of.entry(e.consumer).or_default().push(e.producer); + } + let mut priced: HashMap> = HashMap::new(); + for node in &dag.nodes { + let views: Vec> = producers_of + .get(&node.id) + .map_or(&[][..], Vec::as_slice) + .iter() + .map(|p| { + let produced: &NodeUsage<'a> = &priced[p]; + ProducerOutput { + payload: &nodes[p].payload, + output_schema: &nodes[p].output_schema, + output: produced.output, + sources: produced.sources.clone(), + family: produced.family, + unchanged_columns: produced.unchanged_columns.clone(), + } + }) + .collect(); + let node_priced = node_usage( + node, + &views, + multipliers[&node.id], + retained_tumbling_windows(node, &retention_ms), + (node.output_state.timing == ExecutionTiming::IngestionTime) + .then(|| maintained_window_ms(node.id, nodes, &producers_of)) + .flatten(), + data, + settings, + formula_scale, + ) + .map_err(|error| (node.id, error))?; + priced.insert(node.id, node_priced); + } + Ok(EachNodeUsage { + priced, + multipliers, + reach, + }) +} + +/// The CPU cores a candidate needs at its peak: enough for its +/// ingestion-time work every second (none without any), plus one per +/// query with query-time work, all evaluating at once; at least the cores +/// its average query-time work fills. +fn peak_cpu_cores( + dag: &PhysicalASAPDAG, + nodes: &HashMap, + work_ms: &HashMap, + multipliers: &HashMap, + latency: &[QueryLatency], + request_ms_per_second: f64, +) -> u32 { + let mut ingestion_ms_per_second = 0.0; + // Request overhead is attributed in per_node reports, not work_ms, and so + // must enter the throughput floor separately (including literal queries). + let mut query_ms_per_second = request_ms_per_second; + for node in &dag.nodes { + let per_second = multipliers[&node.id].1; + if nodes[&node.id].output_state.timing == ExecutionTiming::IngestionTime { + ingestion_ms_per_second += work_ms[&node.id] * per_second; + } else { + query_ms_per_second += work_ms[&node.id] * per_second; + } + } + let cores = |ms_per_second: f64| (ms_per_second / 1_000.0).ceil() as u32; + let evaluating = latency.iter().filter(|l| l.query_ms > 0.0).count() as u32; + cores(ingestion_ms_per_second) + evaluating.max(cores(query_ms_per_second)) +} + +/// Each query's latency per evaluation (module doc, "Latency"): the sum of +/// the query-time work it reaches, each node once, plus the longest +/// compaction among the ingestion-time states it reads. Compacting a +/// state's newest window merges, for each of its `I` states per window, +/// the `k` ingest workers' partial copies, `(k − 1)·I` merges, where `k` +/// is the cores its ingestion work takes, at least one. Each merge takes +/// the state's measured merge time (`merge_ms`) when the measured table has +/// one, and one operation otherwise. +fn query_latencies( + dag: &PhysicalASAPDAG, + nodes: &HashMap, + output: &HashMap, + work_ms: &HashMap, + merge_ms: &HashMap, + formula_merge_ms: f64, +) -> Vec { + let inputs = |id: PhysicalASAPNodeId| { + dag.edges + .iter() + .filter(move |e| e.consumer == id) + .map(|e| e.producer) + }; + let at_ingestion = + |id: PhysicalASAPNodeId| nodes[&id].output_state.timing == ExecutionTiming::IngestionTime; + let compaction_ms = |state: PhysicalASAPNodeId| { + let workers = ingest_workers(dag, work_ms, state) as f64; + let states = (output[&state].rows / tumbling_windows_of(nodes[&state])) as f64; + let per_merge_ms = merge_ms.get(&state).copied().unwrap_or(formula_merge_ms); + (workers - 1.0) * states * per_merge_ms + }; + query_starts(dag) + .into_iter() + .map(|start| { + let mut latency = QueryLatency { + compaction_ms: 0.0, + query_ms: 0.0, + }; + let mut seen = HashSet::new(); + let mut stack = start; + while let Some(id) = stack.pop() { + if !seen.insert(id) { + continue; + } + if at_ingestion(id) { + latency.compaction_ms = latency.compaction_ms.max(compaction_ms(id)); + continue; + } + latency.query_ms += work_ms[&id]; + stack.extend(inputs(id)); + } + latency + }) + .collect() +} + +/// The cores `state`'s ingestion work takes, at least one: the ingest +/// workers that each build a partial copy of its newest window. +fn ingest_workers( + dag: &PhysicalASAPDAG, + work_ms: &HashMap, + state: PhysicalASAPNodeId, +) -> u64 { + let mut ingest_ms_per_s = 0.0; + let mut seen = HashSet::new(); + let mut stack = vec![state]; + while let Some(id) = stack.pop() { + if seen.insert(id) { + ingest_ms_per_s += work_ms[&id]; + stack.extend( + dag.edges + .iter() + .filter(|e| e.consumer == id) + .map(|e| e.producer), + ); + } + } + (ingest_ms_per_s / 1_000.0).ceil().max(1.0) as u64 +} diff --git a/crates/cost/src/execution_demand.rs b/crates/cost/src/plan/execution_demand.rs similarity index 85% rename from crates/cost/src/execution_demand.rs rename to crates/cost/src/plan/execution_demand.rs index 6a7e380bd..b5220bd3e 100644 --- a/crates/cost/src/execution_demand.rs +++ b/crates/cost/src/plan/execution_demand.rs @@ -3,74 +3,75 @@ //! This preserves the configured accounting policy, not an exact timestamp //! union: different fixed intervals add, unknown phases join a known phase, //! estimated rates add, and one-off/scheduled demand uses a batch maximum. -//! Call `schedule_groups` or `evaluation_rate` on the chosen reader subset +//! Call `schedule_groups` or `evaluation_rate` on the chosen query subset //! only; neither enumerates subsets. For retained query-time state pass `None` for each range evaluation //! to price once per request, as `computations` does. -use super::analytical::AnalyticalCostError; -use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; -use asap_types::physical::Materialization; +use crate::node::analytical::AnalyticalCostError; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::stages::Lifetime; use asap_types::workload::{QueryRecurrence, RangeEvaluation, RepeatedDemand}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; /// How a [`ScheduleGroup`]'s evaluations add to the others'. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Schedule { - /// Repeating readers on one interval and phase: evaluated together, + /// Repeating queries on one interval and phase: evaluated together, /// once per distinct instant their steps fall on. FixedInterval, - /// One reader with an estimated rate: its rate times its steps. + /// One query with an estimated rate: its rate times its steps. EstimatedRate, - /// The one-off, scheduled and unknown readers, run as one batch: the + /// The one-off, scheduled and unknown queries, run as one batch: the /// most evaluations among them, spread over the horizon. NonRepeating, } -/// Readers of one node that are evaluated together, and how often the node +/// Queries of one node that are evaluated together, and how often the node /// is computed for them. Groups add ([`evaluation_rate`]). #[derive(Debug, Clone, PartialEq)] pub struct ScheduleGroup { - /// The readers in the group, as positions in the reader list given to + /// The queries in the group, as positions in the query list given to /// [`schedule_groups`], ascending. - pub readers: Vec, - /// How often the node is computed per second for these readers. + pub queries: Vec, + /// How often the node is computed per second for these queries. pub evaluations_per_second: f64, /// Which kind of group it is. pub schedule: Schedule, } -/// One fixed (interval, phase) schedule's readers: each step in ms they -/// use with the longest span in ms at that step, and the readers. +/// One fixed (interval, phase) schedule's queries: each step in ms they +/// use with the longest span in ms at that step, and the queries. type FixedSchedule = (BTreeMap, Vec); -/// The groups of `readers` evaluated together (each a recurrence with its +/// The groups of `queries` evaluated together (each a recurrence with its /// range evaluation), in the order [`evaluation_rate`] adds them: fixed /// intervals by (interval, phase), then each estimated rate, then the /// non-repeating batch. A request with a range evaluation is /// [`RangeEvaluation::evaluations_per_request`] evaluations, one without it -/// one. Repeating readers with equal intervals and equal phases form one +/// one. Repeating queries with equal intervals and equal phases form one /// group, every request ending at the same instant: a node they share is /// computed once per distinct instant their steps fall on, -/// `end − k · step_ms` for `k · step_ms ≤ span_ms`. A reader of unknown +/// `end − k · step_ms` for `k · step_ms ≤ span_ms`. A query of unknown /// phase joins the group of the first known phase on its interval, if any. -/// A reader evaluated once per request (no range, or one step) evaluates +/// A query evaluated once per request (no range, or one step) evaluates /// only at the end. Different intervals or phases are different groups (an /// over-estimate for intervals whose instants coincide, e.g. every 20 s and /// every 30 s meet every 60 s); each estimated rate is its own group. -/// One-off, scheduled and unknown readers form one batch group: the most +/// One-off, scheduled and unknown queries form one batch group: the most /// evaluations among them, spread over `non_repeating_query_horizon_seconds`. pub fn schedule_groups<'r>( - readers: impl IntoIterator)>, + queries: impl IntoIterator)>, non_repeating_query_horizon_seconds: f64, ) -> Result, AnalyticalCostError> { // Per interval in ms and phase in ms on it (`None`: unknown), each step - // in ms its readers use and the longest span in ms at that step, with - // the readers; once per request is the interval with span 0. + // in ms its queries use and the longest span in ms at that step, with + // the queries; once per request is the interval with span 0. let mut by_schedule: BTreeMap<(u32, Option), FixedSchedule> = BTreeMap::new(); let mut estimated = Vec::new(); let mut invocations = 0u64; let mut non_repeating = Vec::new(); - for (reader, (recurrence, range_evaluation)) in readers.into_iter().enumerate() { + for (query, (recurrence, range_evaluation)) in queries.into_iter().enumerate() { let steps = range_evaluation.map_or(1, |range| range.evaluations_per_request()); match recurrence { QueryRecurrence::Repeated( @@ -96,7 +97,7 @@ pub fn schedule_groups<'r>( by_schedule.entry((interval.0, phase_ms)).or_default(); let longest_span_ms = longest_span_ms_by_step_ms.entry(step_ms).or_default(); *longest_span_ms = (*longest_span_ms).max(span_ms); - members.push(reader); + members.push(query); } QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { let rate = estimate.expected_rate.0; @@ -104,28 +105,28 @@ pub fn schedule_groups<'r>( return Err(AnalyticalCostError::InvalidRecurrence); } estimated.push(ScheduleGroup { - readers: vec![reader], + queries: vec![query], evaluations_per_second: rate * steps as f64, schedule: Schedule::EstimatedRate, }); } QueryRecurrence::Repeated(RepeatedDemand::Scheduled(times)) => { invocations = invocations.max(times.len() as u64 * steps); - non_repeating.push(reader); + non_repeating.push(query); } QueryRecurrence::OneTime { invocations: count, .. } => { invocations = invocations.max(*count * steps); - non_repeating.push(reader); + non_repeating.push(query); } QueryRecurrence::Unknown => { invocations = invocations.max(steps); - non_repeating.push(reader); + non_repeating.push(query); } } } - // Readers of unknown phase run with the first known phase on their + // Queries of unknown phase run with the first known phase on their // interval. let unknown_phase_intervals_ms: Vec = by_schedule .keys() @@ -152,8 +153,8 @@ pub fn schedule_groups<'r>( let mut groups: Vec = by_schedule .into_iter() .map( - |((interval_ms, _), (longest_span_ms_by_step_ms, readers))| ScheduleGroup { - readers, + |((interval_ms, _), (longest_span_ms_by_step_ms, queries))| ScheduleGroup { + queries, evaluations_per_second: 1_000.0 / f64::from(interval_ms) * distinct_instants_per_request(&longest_span_ms_by_step_ms) as f64, schedule: Schedule::FixedInterval, @@ -163,7 +164,7 @@ pub fn schedule_groups<'r>( groups.extend(estimated); if !non_repeating.is_empty() { groups.push(ScheduleGroup { - readers: non_repeating, + queries: non_repeating, evaluations_per_second: invocations as f64 / non_repeating_query_horizon_seconds, schedule: Schedule::NonRepeating, }); @@ -280,7 +281,7 @@ pub fn query_starts(dag: &PhysicalASAPDAG) -> Vec> { .collect() } -/// What computes a `NotMaterialized` node once per its own execution +/// What computes a not-materialized node once per its own execution /// ([`consumers`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Consumer { @@ -290,19 +291,19 @@ pub enum Consumer { Materialized(PhysicalASAPNodeId), } -/// The consumers of each `NotMaterialized` node of `dag` that any consumer +/// The consumers of each not-materialized node of `dag` that any consumer /// reaches ([`computations`]): the queries and the materialized nodes whose /// walk down their inputs reaches it before meeting a materialized node, /// each once; the queries first, in order, then the materialized nodes in /// `dag`'s node order. pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap> { - let materialized: Vec = dag + let materialized_nodes: Vec = dag .nodes .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) + .filter(|n| n.lifetime != Lifetime::None) .map(|n| n.id) .collect(); - let is_materialized: HashSet = materialized.iter().copied().collect(); + let is_materialized: HashSet = materialized_nodes.iter().copied().collect(); let inputs = |id: PhysicalASAPNodeId| { dag.edges .iter() @@ -314,7 +315,7 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap HashMap, ) -> Result, AnalyticalCostError> { let range_of = |query: usize| range_evaluations.get(query).copied().flatten(); - let materialized: HashMap<_, _> = dag + // Each materialized node's timing and lifetime. + let materialized_nodes: HashMap = dag .nodes .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) - .map(|n| (n.id, n.materialization)) + .filter(|node| node.lifetime != Lifetime::None) + .map(|node| (node.id, (node.output_state.timing, node.lifetime))) .collect(); let inputs = |id: PhysicalASAPNodeId| { dag.edges @@ -385,14 +387,12 @@ pub fn computations( } } let mut rate_of_materialized = HashMap::new(); - for (&id, materialization) in &materialized { - // A chain kept across evaluations is built once per request. - let per_request = matches!( - materialization, - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ); - let rate = match materialization { - Materialization::MatAtIngestion(_) => 1.0, + for (&id, &(timing, lifetime)) in &materialized_nodes { + let at_ingestion = timing == ExecutionTiming::IngestionTime; + // A query-time chain materialized across evaluations is built once per request. + let per_request = !at_ingestion && matches!(lifetime, Lifetime::AcrossEvaluations(_)); + let rate = match at_ingestion { + true => 1.0, _ => evaluation_rate( reached.get(&id).into_iter().flatten().filter_map(|&q| { let range = if per_request { None } else { range_of(q) }; @@ -416,9 +416,9 @@ pub fn computations( } // The refresh window of each node refreshed at ingestion. let refresh_windows = dag.refresh_windows( - materialized + materialized_nodes .iter() - .filter(|(_, m)| matches!(m, Materialization::MatAtIngestion(_))) + .filter(|(_, (timing, _))| *timing == ExecutionTiming::IngestionTime) .map(|(&id, _)| id), ); let mut count: HashMap<_, (u64, f64)> = HashMap::new(); @@ -476,20 +476,20 @@ mod tests { // Unknown phases join the first known phase; separate known phases add. #[test] fn chosen_readers_preserve_phase_accounting() { - let readers = [ + let queries = [ fixed(20_000, None), fixed(20_000, Some(5_000)), fixed(20_000, Some(0)), ]; close( - evaluation_rate(readers[..2].iter().map(|r| (r, None)), 3600.0).unwrap(), + evaluation_rate(queries[..2].iter().map(|r| (r, None)), 3600.0).unwrap(), 1.0 / 20.0, ); close( - evaluation_rate(readers.iter().map(|r| (r, None)), 3600.0).unwrap(), + evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), 2.0 / 20.0, ); - // No global subset cache: each caller's selected readers determine the rate. + // No global subset cache: each caller's selected queries determine the rate. close(evaluation_rate(std::iter::empty(), 3600.0).unwrap(), 0.0); } @@ -521,9 +521,9 @@ mod tests { // Different fixed intervals remain additive even if timestamps can coincide. #[test] fn different_intervals_preserve_existing_conservative_rate() { - let readers = [fixed(20_000, Some(0)), fixed(30_000, Some(0))]; + let queries = [fixed(20_000, Some(0)), fixed(30_000, Some(0))]; close( - evaluation_rate(readers.iter().map(|r| (r, None)), 3600.0).unwrap(), + evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), 1.0 / 20.0 + 1.0 / 30.0, ); } diff --git a/crates/cost/src/plan/mod.rs b/crates/cost/src/plan/mod.rs new file mode 100644 index 000000000..4f8b258d7 --- /dev/null +++ b/crates/cost/src/plan/mod.rs @@ -0,0 +1,8 @@ +//! The whole physical plan DAG: how often each node is computed +//! ([`execution_demand`]), the plan's cost (`cost`), the tumbling windows it +//! retains (`retention`) and how plans rank (`ranking`). + +pub(crate) mod cost; +pub mod execution_demand; +pub(crate) mod ranking; +pub(crate) mod retention; diff --git a/crates/cost/src/ranking.rs b/crates/cost/src/plan/ranking.rs similarity index 97% rename from crates/cost/src/ranking.rs rename to crates/cost/src/plan/ranking.rs index 2c1308b6c..42322c75f 100644 --- a/crates/cost/src/ranking.rs +++ b/crates/cost/src/plan/ranking.rs @@ -8,7 +8,7 @@ use crate::{ClusterObjective, CostModel, PlanCost, QueryLatency, ResourceUsage}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PlanRanking { /// Cost models 1 and 2: by cost per second. - Price, + Cost, /// Latency, then peak memory, then disk. Latency, /// Peak memory, then disk, then latency. @@ -18,7 +18,7 @@ pub enum PlanRanking { impl PlanRanking { pub fn of(model: &CostModel) -> Self { match model { - CostModel::UsageOverTime(_) | CostModel::ProvisionedCapacity(_) => Self::Price, + CostModel::UsageOverTime(_) | CostModel::ProvisionedCapacity(_) => Self::Cost, CostModel::ClusterCapacity(limits) => match limits.objective { ClusterObjective::LowestLatency => Self::Latency, ClusterObjective::LeastMemory => Self::Memory, @@ -40,7 +40,7 @@ impl PlanRanking { let memory = || a.peak_memory_bytes.cmp(&b.peak_memory_bytes); let disk = || a.disk_bytes.cmp(&b.disk_bytes); let order = match self { - Self::Price => rounded( + Self::Cost => rounded( cost.cost_per_second.unwrap_or(f64::INFINITY), best.cost_per_second.unwrap_or(f64::INFINITY), ), diff --git a/crates/cost/src/plan/retention.rs b/crates/cost/src/plan/retention.rs new file mode 100644 index 000000000..336358152 --- /dev/null +++ b/crates/cost/src/plan/retention.rs @@ -0,0 +1,134 @@ +//! How long a plan keeps each tumbling window chain and which windows it +//! retains, read from the queries each node reaches. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use crate::plan::execution_demand::query_starts; +use crate::{node_map, NodeMap}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::{ASAPOp, NonASAPOp}; +use asap_types::plan::physical_asap_dag::{ + PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, +}; +use asap_types::stages::Lifetime; +use asap_types::workload::RangeEvaluation; + +/// For each tumbling-window chain of `dag` materialized across +/// evaluations, how far back from the newest evaluation its completed +/// windows must be retained, in milliseconds. `reach` is [`query_reach`] of +/// `dag`; `range_evaluations[q]` is query `q`'s (none when missing). +pub fn tumbling_window_retention_ms( + dag: &PhysicalASAPDAG, + reach: &[HashSet], + range_evaluations: &[Option], +) -> BTreeMap { + let nodes = node_map(dag); + dag.nodes + .iter() + .filter_map(|node| match (&node.payload, node.lifetime) { + ( + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), + Lifetime::AcrossEvaluations(_), + ) => { + let _ = (&nodes, reach, range_evaluations); + todo!("#1207 PR 7: a chain's retention, max(offset + range) over the paths that read it") + } + _ => None, + }) + .collect() +} + +/// The tumbling-window retention of every materialized tumbling-window chain of `dag` +/// ([`tumbling_window_retention_ms`] over [`query_reach`]), for a plan whose query +/// `q` has range evaluation `range_evaluations[q]` (none when missing): +/// what Stage 2's realizer records for a realized plan. +pub fn tumbling_window_retention_ms_of_plan( + dag: &PhysicalASAPDAG, + range_evaluations: &[Option], +) -> BTreeMap { + tumbling_window_retention_ms(dag, &query_reach(dag, &node_map(dag)), range_evaluations) +} + +/// The panes a chain in `retention_ms` ([`tumbling_window_retention_ms`]) keeps +/// completed: its retention over its pane width, at least its pane count. +pub fn retained_tumbling_windows( + node: &asap_types::plan::physical_asap_dag::PhysicalASAPDAGNode, + retention_ms: &BTreeMap, +) -> u64 { + match (&node.payload, retention_ms.get(&node.id)) { + ( + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }), + Some(&retention), + ) => (retention / tumbling_window_width_ms.max(&1)).max(tumbling_windows_of(node)), + _ => tumbling_windows_of(node), + } +} + +/// A node's tumbling windows: a tumbling-window chain's count, 1 for any other node. +pub fn tumbling_windows_of(node: &asap_types::plan::physical_asap_dag::PhysicalASAPDAGNode) -> u64 { + match &node.payload { + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { + todo!("#1207 PR 7: the windows a chain retains (h_v)") + } + _ => 1, + } +} + +/// The nodes each query reaches at query time, in workload order: its walk +/// from its roots, stopping at (and including) ingestion-time nodes. +pub fn query_reach( + dag: &PhysicalASAPDAG, + nodes: &HashMap, +) -> Vec> { + query_starts(dag) + .into_iter() + .map(|mut stack| { + let mut seen = HashSet::new(); + while let Some(id) = stack.pop() { + if seen.insert(id) + && nodes[&id].output_state.timing != ExecutionTiming::IngestionTime + { + stack.extend( + dag.edges + .iter() + .filter(|e| e.consumer == id) + .map(|e| e.producer), + ); + } + } + seen + }) + .collect() +} + +/// The window an ingestion-time grouping `node` maintains: a tumbling-window chain's +/// tumbling windows, or the time range below it (through filters, projections and +/// time shifts). `None` when there is none. +pub fn maintained_window_ms( + node: PhysicalASAPNodeId, + nodes: &NodeMap<'_>, + producers_of: &HashMap>, +) -> Option { + match &nodes[&node].payload { + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { + todo!("#1207 PR 7: the window a chain maintains, its retention (h_v)") + } + Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) => { + return Some(range.as_millis() as u64) + } + _ => {} + } + let producer = *producers_of.get(&node)?.first()?; + match &nodes[&producer].payload { + Payload::NonASAP( + NonASAPOp::TimeRange { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::TimeShift { .. }, + ) => maintained_window_ms(producer, nodes, producers_of), + _ => None, + } +} diff --git a/crates/cost/src/resources.rs b/crates/cost/src/resources.rs deleted file mode 100644 index a13886e33..000000000 --- a/crates/cost/src/resources.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! Resource dimensions of analytical estimates. -//! -//! The CPU payload establishes its unit and operation scope; CPU operations -//! must never be interpreted as nanoseconds. `None` means unavailable, not zero. - -mod cpu; -mod physical; - -pub(crate) use cpu::ModeledCpu; -pub(crate) use physical::PhysicalResources; -#[cfg(test)] -mod tests { - use super::*; - - /// Default resources preserve unavailable dimensions instead of inventing zeros. - #[test] - fn missing_dimensions_remain_unavailable() { - let resources = PhysicalResources::, u64>::default(); - assert_eq!(resources.cpu, None); - assert_eq!(resources.peak_memory_bytes, None); - assert_eq!(resources.retained_memory_bytes, None); - assert_eq!(resources.scan_bytes, None); - assert_eq!(resources.serialized_bytes, None); - assert_eq!(resources.disk_bytes, None); - } - - /// Storage occupancy and scan traffic are independent even though both use bytes. - #[test] - fn byte_dimensions_round_trip_independently() { - let resources = PhysicalResources { - cpu: Some(12.0), - peak_memory_bytes: Some(100), - retained_memory_bytes: Some(70), - scan_bytes: Some(200), - serialized_bytes: Some(40), - disk_bytes: Some(4096), - }; - let json = serde_json::to_value(resources).unwrap(); - let decoded: PhysicalResources, u64> = serde_json::from_value(json).unwrap(); - assert_eq!(decoded, resources); - } -} diff --git a/crates/cost/src/resources/mod.rs b/crates/cost/src/resources/mod.rs new file mode 100644 index 000000000..832670f79 --- /dev/null +++ b/crates/cost/src/resources/mod.rs @@ -0,0 +1,73 @@ +//! Resource dimensions of analytical estimates. +//! +//! The CPU payload establishes its unit and operation scope; CPU operations +//! must never be interpreted as nanoseconds. `None` means unavailable, not zero. + +mod cpu; +mod physical; + +pub(crate) use cpu::ModeledCpu; +pub(crate) use physical::PhysicalResources; + +use crate::QueryLatency; + +/// What one candidate uses, per second of wall time unless the name says +/// otherwise, without prices: its nodes' CPU, scans, retained memory and disk +/// added up, with the candidate's peak cores and per-query latency. Every [`CostModel`](crate::CostModel) reads +/// it. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ResourceUsage { + /// CPU operations per second: ingestion-time and query-time work. + pub cpu_ops_per_second: f64, + /// Bytes scanned per second. + pub scanned_bytes_per_second: f64, + /// Memory held between computations, averaged over time: retained + /// byte-seconds per second (ingestion-time state always, query-time + /// state while its evaluation runs). + pub average_memory_bytes: f64, + /// The most memory live at once, assuming every evaluation overlaps: + /// in-memory materialized state plus each computation's working memory, once + /// per consumer that recomputes it, with the consumers materialized at ingestion time + /// of one refresh window counted once. + pub peak_memory_bytes: u64, + /// Bytes retained on disk: state materialized on `StorageMedium::Disk`, plus the + /// raw data query-time scans read (the longest window of each source). + pub disk_bytes: u64, + /// CPU cores to provision for the peak: the ingestion work's cores plus + /// one per query evaluating at once. + pub peak_cpu_cores: u32, + /// One per query, in workload order: its latency per evaluation. + pub latency_per_query: Vec, +} +#[cfg(test)] +mod tests { + use super::*; + + /// Default resources preserve unavailable dimensions instead of inventing zeros. + #[test] + fn missing_dimensions_remain_unavailable() { + let resources = PhysicalResources::, u64>::default(); + assert_eq!(resources.cpu, None); + assert_eq!(resources.peak_memory_bytes, None); + assert_eq!(resources.retained_memory_bytes, None); + assert_eq!(resources.scan_bytes, None); + assert_eq!(resources.serialized_bytes, None); + assert_eq!(resources.disk_bytes, None); + } + + /// Storage occupancy and scan traffic are independent even though both use bytes. + #[test] + fn byte_dimensions_round_trip_independently() { + let resources = PhysicalResources { + cpu: Some(12.0), + peak_memory_bytes: Some(100), + retained_memory_bytes: Some(70), + scan_bytes: Some(200), + serialized_bytes: Some(40), + disk_bytes: Some(4096), + }; + let json = serde_json::to_value(resources).unwrap(); + let decoded: PhysicalResources, u64> = serde_json::from_value(json).unwrap(); + assert_eq!(decoded, resources); + } +} diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs deleted file mode 100644 index f811145d4..000000000 --- a/crates/cost/src/usage.rs +++ /dev/null @@ -1,1687 +0,0 @@ -//! Costing a physical plan DAG node by node: each node's resource usage -//! ([`node_usage`]), then the plan's usage, latency, retention, kept tumbling window -//! chains and cost ([`plan_cost`]). Moved from Stage 3 (`asap-plan-selection`, -//! whose module doc, "Cost model", describes the model), so Stage 3 computes -//! no cost of its own. - -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; - -use crate::analytical::{ - estimate_operator, AnalyticalCostError, PhysicalOperator, ResourceEstimate, -}; -use crate::execution_demand::{computations, consumers, evaluation_rate, query_starts, Consumer}; -use crate::operator_statistics::{ - EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics, -}; -use crate::state_bytes::{ - keeps_every_value, row_bytes, sketch_shape, summary_shape, StateInputRows, -}; -use crate::{aggregate_lookup, lookup, StateFamily}; -use crate::{ - CalibrationFit, CostInputs, CostModel, NodeCost, NodePriceSource, OperatorKind, PlanCost, - QueryLatency, ResourceUsage, COST_SOURCE, COST_UNIT, MEASURED_COST_SOURCE, MIXED_COST_SOURCE, -}; -use asap_types::ir::operator::{GroupKeys, Reduction, Source}; -use asap_types::ir::physical_export::{ - PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, ScanWindow, -}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::schema::{ - FieldDataType, GroupingStrategy, Schema, SketchParams, SketchStatistic, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, TimeRangeKind}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{DataWorkload, QueryRecurrence, RangeEvaluation}; - -use asap_types::workload::bounds::{ - declared_non_negative, declared_series, group_count_bound, source_name, source_rows_per_ms, - unchanged_source_columns, -}; - -/// Measure every node of `dag` per second of wall time (module doc, "Cost -/// model") and price it under `settings.cost_model`. Nodes are exported -/// children first, so each node's input statistics are known when it is -/// reached. A query-time node is measured per computation over the rows its -/// time ranges keep, times how often it is computed per second -/// ([`computations`]); an ingestion-time node over one second of arriving -/// rows. A materialized node's kept state is counted in byte-seconds per -/// second; a tumbling-window chain kept beyond one evaluation keeps the tumbling windows its -/// retention covers ([`tumbling_window_retention_ms`]). `nodes` and `kept_chains` are -/// `dag`'s ([`node_map`], [`kept_tumbling_window_chains`]); `recurrence[q]` and -/// `range_evaluations[q]` say how often query `q` runs and how many steps -/// each run evaluates (one when missing). -#[allow(clippy::too_many_arguments)] -pub fn plan_cost( - dag: &PhysicalASAPDAG, - nodes: &NodeMap<'_>, - kept_chains: &HashSet, - recurrence: &[QueryRecurrence], - range_evaluations: &[Option], - data: &DataWorkload, - settings: &CostInputs, - formula_scale: f64, -) -> Result<(PlanCost, CalibrationFit), (PhysicalASAPNodeId, AnalyticalCostError)> { - let first = dag.nodes.first().map(|n| n.id).unwrap_or_default(); - let table = &settings.measured_operation_costs; - let mut fit = CalibrationFit::default(); - let speed = settings.resource_speed; - // Cost model 1 prices each node as it is measured. - let usage_prices = match &settings.cost_model { - CostModel::UsageOverTime(prices) => Some(prices), - _ => None, - }; - let EachNodeUsage { - priced, - multipliers, - reach, - } = each_node_usage( - dag, - nodes, - kept_chains, - recurrence, - range_evaluations, - data, - settings, - formula_scale, - None, - None, - )?; - let mut output: HashMap = HashMap::new(); - let mut per_node = BTreeMap::new(); - // The candidate's usage, accumulated node by node; peak cores and - // latency are filled in once every node is measured. - let mut usage = ResourceUsage::default(); - // Each node's work in ms on one core: one computation at query time, - // one second of arriving rows at ingestion time. - let mut work_ms: HashMap = HashMap::new(); - // Query-time memory held only while one evaluation runs, a MatAtQuery - // state in memory or a NotMaterialized node's working memory: (bytes, - // per second, whether each consumer computes its own copy), - // counted once the evaluations' latencies are known, in node order so - // the sum is the same in every run. - let mut held: BTreeMap = BTreeMap::new(); - // The raw data query-time scans read, kept on disk: per source, the - // largest scan's (bytes, node). - let mut raw_data: HashMap<&Source, (u64, PhysicalASAPNodeId)> = HashMap::new(); - // Each in-memory ingestion-time state's bytes per window (what it keeps - // for one window, its output or an exact aggregate's values). - let mut maintained: Vec<(PhysicalASAPNodeId, u64)> = Vec::new(); - // Each summary node's state family, to look up in the measured table: - // a build's own (one sketch instance per group; a shared Hydra - // structure is not that sketch), inherited by the merges, estimates and - // finalizations over it. - let mut family_of: HashMap> = HashMap::new(); - let sketch_size = - |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; - for node in &dag.nodes { - let node_priced = &priced[&node.id]; - usage.cpu_ops_per_second += node_priced.cpu_ops_per_second; - usage.scanned_bytes_per_second += node_priced.scanned_bytes_per_second; - usage.average_memory_bytes += node_priced.average_memory_bytes; - usage.peak_memory_bytes = usage - .peak_memory_bytes - .saturating_add(node_priced.peak_memory_bytes); - usage.disk_bytes = usage.disk_bytes.saturating_add(node_priced.disk_bytes); - fit.add(node_priced.fit); - work_ms.insert(node.id, node_priced.work_ms_per_execution); - output.insert(node.id, node_priced.output); - if node_priced.family != StateFamily::None { - family_of.insert(node.id, node_priced.family); - } - if let Some(entry) = node_priced.held { - held.insert(node.id, entry); - } - if let Some(window) = node_priced.maintained { - maintained.push((node.id, window)); - } - if let Some((source, bytes)) = node_priced.raw_scan { - let largest = raw_data.entry(source).or_insert((bytes, node.id)); - if bytes > largest.0 { - *largest = (bytes, node.id); - } - } - per_node.insert(node.id, node_priced.cost.clone()); - } - // The runtime's fixed cost per query request, charged to the node the - // query reads first and added to its latency below. - let evaluation_ms = table.ns_per_evaluation * 1e-6; - let starts = query_starts(dag); - let mut evaluation_ms_per_query = vec![0.0; starts.len()]; - let mut request_ms_per_second = 0.0; - // Literal-only roots have no operator to own their fee. Keep that charge - // at plan level while preserving existing attribution for operator queries. - let mut unattached_request_cost = 0.0; - let mut has_unattached_request_overhead = false; - for (query, start) in starts.iter().enumerate() { - if evaluation_ms <= 0.0 { - continue; - } - let diagnostic_node = start.first().copied().unwrap_or(first); - let requests_per_second = evaluation_rate( - recurrence.get(query).map(|r| (r, None)), - settings.non_repeating_query_horizon_seconds, - ) - .map_err(|error| (diagnostic_node, error))?; - let ops_per_second = evaluation_ms / speed.ms_per_cpu_op * requests_per_second; - usage.cpu_ops_per_second += ops_per_second; - request_ms_per_second += evaluation_ms * requests_per_second; - if !usage.cpu_ops_per_second.is_finite() || !request_ms_per_second.is_finite() { - return Err((diagnostic_node, AnalyticalCostError::Overflow)); - } - evaluation_ms_per_query[query] = evaluation_ms; - let Some(&first_read) = start.first() else { - has_unattached_request_overhead = true; - if let Some(prices) = usage_prices { - unattached_request_cost += prices.cost_per_cpu_op * ops_per_second; - if !unattached_request_cost.is_finite() { - return Err((diagnostic_node, AnalyticalCostError::Overflow)); - } - } - continue; - }; - let node = per_node - .get_mut(&first_read) - .expect("every node is measured"); - node.cpu_core_seconds_per_second += evaluation_ms * requests_per_second / 1_000.0; - if !node.cpu_core_seconds_per_second.is_finite() { - return Err((first_read, AnalyticalCostError::Overflow)); - } - node.detail += &format!( - "; q{} request overhead {:.3} ns x {requests_per_second:.4}/s", - query + 1, - table.ns_per_evaluation - ); - if let (Some(cost), Some(prices)) = (&mut node.cost_per_second, usage_prices) { - *cost += prices.cost_per_cpu_op * ops_per_second; - if !cost.is_finite() { - return Err((first_read, AnalyticalCostError::Overflow)); - } - } - } - for (bytes, id) in raw_data.into_values() { - let node = per_node.get_mut(&id).expect("every node is measured"); - usage.disk_bytes = usage.disk_bytes.saturating_add(bytes); - node.detail += &format!("; keeps {bytes} bytes of raw data on disk"); - if let (Some(cost), Some(prices)) = (&mut node.cost_per_second, usage_prices) { - *cost += prices.cost_per_disk_byte_second * bytes as f64; - } - } - // With k ingest workers, each of the k − 1 beyond the first builds its - // own partial copy of an in-memory ingestion-time state's newest window - // (merged at query time, [`query_latencies`]), held at the peak too. - for (id, window) in maintained { - let copies = ingest_workers(dag, &work_ms, id) - 1; - if copies == 0 { - continue; - } - let bytes = copies.saturating_mul(window); - let measured = per_node.get_mut(&id).expect("every node is measured"); - usage.peak_memory_bytes = usage.peak_memory_bytes.saturating_add(bytes); - measured.detail += - &format!("; {copies} partial copies of its newest window, {bytes} bytes"); - } - // Milliseconds of one merge of each state with a measured query-time - // merge entry (measured ns · 10⁻⁶), for the newest window's compaction. - let merge_ms: HashMap = family_of - .iter() - .filter_map(|(id, family)| { - lookup( - table, - OperatorKind::SummaryMerge, - ExecutionTiming::QueryTime, - *family, - sketch_size, - ) - .map(|unit| (*id, (unit.ns_per_unit + unit.ns_per_invocation) * 1e-6)) - }) - .collect(); - let mut latency = query_latencies( - dag, - nodes, - &output, - &work_ms, - &merge_ms, - speed.ms_per_cpu_op * formula_scale, - ); - for (latency, evaluation_ms) in latency.iter_mut().zip(evaluation_ms_per_query) { - latency.query_ms += evaluation_ms; - } - // Query-time memory is held for one evaluation: a MatAtQuery state from - // its build until the longest query reading it finishes, about that - // query's latency, once per execution. A NotMaterialized node's working - // memory is computed once per consumer ([`consumers`]), and each copy is - // held while its consumer's evaluation runs: a query's for that query's - // latency, at its rate; a materialized node's until the longest query - // reading that node finishes, at the node's rate, as its MatAtQuery - // state is. - let longest_reader_ms = |id: PhysicalASAPNodeId| { - reach - .iter() - .zip(&latency) - .filter(|(reached, _)| reached.contains(&id)) - .map(|(_, l)| l.total_ms()) - .fold(0.0, f64::max) - }; - let consumers_of = consumers(dag); - for (id, (state, per_second, computed_per_consumer)) in held { - // The milliseconds it is held per second: each execution's held - // time times its executions per second, summed. - let held_ms_per_second = match computed_per_consumer { - false => per_second * longest_reader_ms(id), - true => { - let mut sum = 0.0; - for &consumer in consumers_of.get(&id).into_iter().flatten() { - sum += match consumer { - Consumer::Query(query) => { - let rate = evaluation_rate( - recurrence - .get(query) - .map(|r| (r, range_evaluations.get(query).copied().flatten())), - settings.non_repeating_query_horizon_seconds, - ) - .map_err(|error| (id, error))?; - rate * latency.get(query).map_or(0.0, QueryLatency::total_ms) - } - Consumer::Materialized(node) => { - multipliers[&node].1 * longest_reader_ms(node) - } - }; - } - sum - } - }; - let evaluation_ms = match per_second > 0.0 { - true => held_ms_per_second / per_second, - false => 0.0, - }; - let byte_seconds = state as f64 * held_ms_per_second / 1_000.0; - let node = per_node.get_mut(&id).expect("every node is measured"); - usage.average_memory_bytes += byte_seconds; - node.detail += &format!("; holds {state} bytes for {evaluation_ms:.1} ms"); - if let (Some(cost), Some(prices)) = (&mut node.cost_per_second, usage_prices) { - *cost += prices.cost_per_memory_byte_second * byte_seconds; - if !cost.is_finite() { - return Err((id, AnalyticalCostError::Overflow)); - } - } - } - usage.peak_cpu_cores = peak_cpu_cores( - dag, - nodes, - &work_ms, - &multipliers, - &latency, - request_ms_per_second, - ); - usage.latency_per_query = latency; - let cost = match &settings.cost_model { - CostModel::UsageOverTime(_) => { - let node_cost: Option = per_node.values().map(|node| node.cost_per_second).sum(); - let total = node_cost.map(|cost| cost + unattached_request_cost); - if total.is_some_and(|cost| !cost.is_finite()) { - return Err((first, AnalyticalCostError::Overflow)); - } - total - } - CostModel::ProvisionedCapacity(prices) => { - let cost = prices.cost_per_provisioned_memory_byte_second - * usage.peak_memory_bytes as f64 - + prices.cost_per_provisioned_cpu_core_second * f64::from(usage.peak_cpu_cores) - + prices.cost_per_disk_byte_second * usage.disk_bytes as f64; - if !cost.is_finite() { - return Err((first, AnalyticalCostError::Overflow)); - } - Some(cost) - } - CostModel::ClusterCapacity(_) => None, - }; - Ok(( - PlanCost { - usage, - cost_per_second: cost, - cost_unit: COST_UNIT, - source: if has_unattached_request_overhead { - if per_node - .values() - .all(|node| node.price_source == NodePriceSource::Measured) - { - MEASURED_COST_SOURCE - } else { - MIXED_COST_SOURCE - } - } else { - cost_source(&per_node) - }, - per_node, - }, - fit, - )) -} - -/// Every node of a physical DAG priced ([`each_node_usage`]), with each -/// node's `(multiplier, per_second)` ([`computations`]) and the nodes each -/// query reaches at query time ([`query_reach`]). -struct EachNodeUsage<'a> { - priced: HashMap>, - multipliers: HashMap, - reach: Vec>, -} - -/// [`node_usage`] over every node of `dag`, children first, with the -/// arguments of [`plan_cost`]. `root_scan_lookback_ms` is the -/// look-back of a query-time scan no window above it in `dag` bounds (a -/// scan priced without its consumers); `None` when the scan has no -/// window, so it reads the source's `rows_held` or fails with -/// `MissingSourceFact`, as Stage 3 does. `root_rates`, when given, are -/// the computations per second of each root's consumer in place of its -/// query's own rate ([`computations`]). -#[allow(clippy::too_many_arguments)] -fn each_node_usage<'a>( - dag: &'a PhysicalASAPDAG, - nodes: &NodeMap<'a>, - kept_chains: &HashSet, - recurrence: &[QueryRecurrence], - range_evaluations: &[Option], - data: &DataWorkload, - settings: &'a CostInputs, - formula_scale: f64, - root_scan_lookback_ms: Option, - root_rates: Option<&[f64]>, -) -> Result, (PhysicalASAPNodeId, AnalyticalCostError)> { - let first = dag.nodes.first().map(|n| n.id).unwrap_or_default(); - let multipliers = computations( - dag, - recurrence, - range_evaluations, - settings.non_repeating_query_horizon_seconds, - root_rates, - ) - .map_err(|error| (first, error))?; - let reach = query_reach(dag, nodes); - let retention_ms = tumbling_window_retention_ms(dag, &reach, range_evaluations); - let mut producers_of: HashMap> = HashMap::new(); - for e in &dag.edges { - producers_of.entry(e.consumer).or_default().push(e.producer); - } - let mut priced: HashMap> = HashMap::new(); - for node in &dag.nodes { - let views: Vec> = producers_of - .get(&node.id) - .map_or(&[][..], Vec::as_slice) - .iter() - .map(|p| { - let produced: &NodeUsage<'a> = &priced[p]; - ProducerOutput { - payload: &nodes[p].payload, - output_schema: &nodes[p].output_schema, - output: produced.output, - sources: produced.sources.clone(), - family: produced.family, - unchanged_columns: produced.unchanged_columns.clone(), - } - }) - .collect(); - let scan_window = match (&node.payload, node.output_state.timing) { - (Payload::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => { - match dag.scan_window(node.id, kept_chains) { - // Priced as before #955 decides `@` anchors and - // unwindowed readers: the widest window read. - ScanWindow::Unbounded => trailing_window( - widest_window_read_ms(dag, nodes, node.id, kept_chains) - .or(root_scan_lookback_ms), - ), - window => window, - } - } - _ => ScanWindow::Unbounded, - }; - let node_priced = node_usage( - node, - &views, - multipliers[&node.id], - scan_window, - retained_tumbling_windows(node, &retention_ms), - (node.output_state.timing == ExecutionTiming::IngestionTime) - .then(|| maintained_window_ms(node.id, nodes, &producers_of)) - .flatten(), - data, - settings, - formula_scale, - ) - .map_err(|error| (node.id, error))?; - priced.insert(node.id, node_priced); - } - Ok(EachNodeUsage { - priced, - multipliers, - reach, - }) -} - -/// The state family of a node with `payload` whose first producer has -/// `producer_family`: a build's own (one sketch instance per group; a -/// shared Hydra structure is not that sketch), inherited by the merges, -/// estimates and finalizations over it, none otherwise. -fn state_family<'a>( - payload: &'a Payload, - producer_family: Option>, -) -> StateFamily<'a> { - match payload { - Payload::ASAP( - ASAPOp::SummaryAgg { - family, grouping, .. - } - | ASAPOp::SummaryTumblingWindowChain { - family, grouping, .. - }, - ) => match family { - FieldDataType::Sketch(kind, _) - if *grouping == GroupingStrategy::PerSubpopulationInstance => - { - StateFamily::Sketch(kind.params()) - } - FieldDataType::ExactAggregate(kind, _) => StateFamily::Exact(kind), - _ => StateFamily::None, - }, - Payload::ASAP( - ASAPOp::SummaryMerge { .. } - | ASAPOp::SummaryEstimate { .. } - | ASAPOp::FinalizeExactAccumulator { .. }, - ) => producer_family.unwrap_or(StateFamily::None), - _ => StateFamily::None, - } -} - -/// One producer of a node being priced ([`node_usage`]): its payload and -/// schema, its output statistics, the sources it reads and its state -/// family. -#[derive(Debug, Clone)] -pub struct ProducerOutput<'a> { - pub payload: &'a Payload, - pub output_schema: &'a Schema, - pub output: EdgeStatistics, - pub sources: BTreeSet<&'a str>, - pub family: StateFamily<'a>, - /// Its output columns that still hold one source row's label (or - /// column) values unchanged ([`unchanged_source_columns`]). - pub unchanged_columns: BTreeSet<&'a str>, -} - -/// What one node uses and outputs ([`node_usage`]): its output statistics, sources and state family for its -/// consumers; its work for one computation (one execution) in ms on one -/// core; its usage per second; what is charged -/// once the whole plan is known (memory held for a query's evaluation, an -/// in-memory ingestion window's partial copies, the raw data a query-time -/// scan keeps on disk); its calibration fit and its [`NodeCost`]. -#[derive(Debug, Clone)] -pub struct NodeUsage<'a> { - pub output: EdgeStatistics, - pub sources: BTreeSet<&'a str>, - pub family: StateFamily<'a>, - /// Its output columns that still hold one source row's label (or - /// column) values unchanged ([`unchanged_source_columns`]). - pub unchanged_columns: BTreeSet<&'a str>, - pub work_ms_per_execution: f64, - pub cpu_ops_per_second: f64, - pub scanned_bytes_per_second: f64, - pub average_memory_bytes: f64, - pub peak_memory_bytes: u64, - pub disk_bytes: u64, - /// Query-time memory held while an evaluation runs: (bytes, computations - /// per second, whether each consumer computes its own copy, held while - /// that consumer's evaluation runs ([`plan_cost`]), rather than one copy - /// held until the longest query reading the node finishes). - pub held: Option<(u64, f64, bool)>, - /// An in-memory ingestion-time state's bytes per window. - pub maintained: Option, - /// A query-time scan's source and the raw bytes it reads. - pub raw_scan: Option<(&'a Source, u64)>, - pub fit: CalibrationFit, - pub cost: NodeCost, -} - -/// Measure and price one node of a physical DAG (one step of -/// [`plan_cost`]): `producers` are its producers in edge order, -/// `rate` its `(multiplier, per_second)` ([`computations`]), `scan_window` -/// the raw rows a query-time scan reads ([`PhysicalASAPDAG::scan_window`]; -/// read only for scans: it reads `width_ms` of rows and keeps them back to -/// [`ScanWindow::lookback_ms`]), `retained_tumbling_window_count` the tumbling windows a tumbling-window chain keeps -/// completed (`retained_tumbling_windows`), and `ingestion_window_ms` the window an -/// ingestion-time node maintains (`maintained_window_ms`). -#[allow(clippy::too_many_arguments)] -pub fn node_usage<'a>( - node: &'a asap_types::ir::physical_export::PhysicalASAPDAGNode, - producers: &[ProducerOutput<'a>], - rate: (u64, f64), - scan_window: ScanWindow, - retained_tumbling_window_count: u64, - ingestion_window_ms: Option, - data: &DataWorkload, - settings: &'a CostInputs, - formula_scale: f64, -) -> Result, AnalyticalCostError> { - let table = &settings.measured_operation_costs; - let speed = settings.resource_speed; - let usage_prices = match &settings.cost_model { - CostModel::UsageOverTime(prices) => Some(prices), - _ => None, - }; - let sketch_size = - |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; - let mut fit = CalibrationFit::default(); - let mut held = None; - let mut maintained = None; - let mut raw_scan = None; - // A query-time scan's raw rows kept ([`ScanWindow::lookback_ms`]). - let mut raw_kept_rows = None; - let ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; - let inputs: Vec = producers.iter().map(|p| p.output).collect(); - let sources: BTreeSet<&str> = match &node.payload { - Payload::NonASAP(NonASAPOp::Scan { source, .. }) => [source_name(source)].into(), - _ => producers - .iter() - .flat_map(|p| p.sources.iter().copied()) - .collect(), - }; - // The series its sources declare, when they all do. - let series = declared_series(&sources, data); - // The rows per ms its sources receive, which its scans and time - // ranges read, when the data declares an arrival rate. - let rows_per_ms = source_rows_per_ms(&sources, data); - // The fact a rate needs, named when it is missing. - let missing_rate = |source: &str| AnalyticalCostError::MissingSourceFact { - source_name: source.to_string(), - missing: "an arrival rate (the arrival's rows_per_second, or the \ - source's series_count with sample_interval)", - }; - let family = state_family(&node.payload, producers.first().map(|p| p.family)); - let unchanged_columns = unchanged_source_columns( - node, - &producers - .iter() - .map(|p| (p.output_schema, &p.unchanged_columns)) - .collect::>(), - ); - let operator = OperatorKind::of(&node.payload); - let unit_price = match &node.payload { - Payload::NonASAP(NonASAPOp::Aggregate { measures, .. }) => { - aggregate_lookup(table, node.output_state.timing, measures) - } - _ => lookup( - table, - operator, - node.output_state.timing, - family, - sketch_size, - ), - }; - // The kernel's measured bytes per state, when the table has them. - let measured_state_bytes = unit_price - .and_then(|p| p.memory_bytes_per_state) - .map(|bytes| bytes.round() as u64); - let mut price_source = NodePriceSource::Analytical; - let input = inputs - .first() - .copied() - .unwrap_or(EdgeStatistics { rows: 1, bytes: 1 }); - let width = row_bytes(&node.output_schema); - let edge = |rows: u64| -> Result { - Ok(EdgeStatistics { - rows, - bytes: rows - .checked_mul(width) - .ok_or(AnalyticalCostError::Overflow)?, - }) - }; - // Whether a summary built here sees raw data samples (its producer a - // scan or a time range) of sources all declared non-negative. - let raw_data_samples_non_negative = || { - let raw = producers.first().is_some_and(|producer| { - matches!( - producer.payload, - Payload::NonASAP(NonASAPOp::Scan { .. } | NonASAPOp::TimeRange { .. }) - ) - }); - raw && sources - .iter() - .all(|source| declared_non_negative(data, source)) - }; - let unary = |output| UnaryEdgeStatistics { - input, - output, - promql: None, - }; - // The schema the node's group and partition keys index. - let input_schema = producers.first().map(|p| p.output_schema); - // The rows a grouping reads per evaluation: its input at query - // time; at ingestion time, the arriving rows of the window it - // maintains (one second of them does not bound its groups). - // Query time always has its input rows; an ingestion-time grouping - // with no window below it is an error (Stage 2 builds none). - let rows_read = || match ingestion { - false => Ok(input.rows.max(1)), - true => { - let window_ms = - ingestion_window_ms.ok_or(AnalyticalCostError::IngestionGroupingWithoutWindow)?; - // Name a source with no rate of its own. - let rows_per_ms = rows_per_ms.ok_or_else(|| { - let unrated = sources - .iter() - .find(|source| source_rows_per_ms(&[**source].into(), data).is_none()); - missing_rate(unrated.copied().unwrap_or("(none)")) - })?; - Ok(((rows_per_ms * window_ms as f64).round() as u64).max(1)) - } - }; - let estimated_groups = |keys: &GroupKeys| { - let rows_read = rows_read()?; - Ok(group_count_bound( - keys, - input_schema, - &sources, - producers - .first() - .map_or(&BTreeSet::new(), |producer| &producer.unchanged_columns), - data, - rows_read, - )) - }; - let groups = |reduction: &Reduction| match reduction { - Reduction::Reduce(keys) if keys.keys().is_empty() && !keys.is_without() => Ok(1), - Reduction::Reduce(keys) => estimated_groups(keys), - // One group per series: the declared series (at query time at - // most the rows read), else the rows read, each series having at - // least one. - Reduction::PerEntity => match (ingestion, series) { - (true, Some(series)) => Ok(series.max(1)), - (false, Some(series)) => Ok(series.min(input.rows).max(1)), - (_, None) => rows_read(), - }, - }; - let (out, estimate, detail) = match &node.payload { - Payload::NonASAP(operator) => match operator { - NonASAPOp::Scan { source, .. } => { - // At ingestion time, one second of arriving rows; at - // query time, the arriving rows of the window read above - // it, at most the rows the source holds (all of them - // without a window). - let source = source_name(source); - let rows_held = data.sources.get(source).and_then(|facts| facts.rows_held); - let window_width_ms = match scan_window { - ScanWindow::Within { width_ms, .. } => { - Some(u64::try_from(width_ms).unwrap_or(0)) - } - ScanWindow::Unbounded => None, - }; - let lookback = match ingestion { - true => Some(1_000), - false => window_width_ms, - }; - let by_rate = |lookback: Option| { - rows_per_ms.zip(lookback).map(|(rows_per_ms, lookback)| { - (rows_per_ms * lookback as f64).round() as u64 - }) - }; - // The raw rows kept for a query-time scan: back to the - // oldest row it reads, `end_offset + width` (an offset - // window keeps more than it reads), at most those held. - if !ingestion { - let kept_rows = match (by_rate(scan_window.lookback_ms()), rows_held) { - (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), - (by_rate, rows_held) => by_rate.or(rows_held), - }; - raw_kept_rows = kept_rows.map(|rows| rows.max(1)); - } - let by_rate = by_rate(lookback); - let rows = match (ingestion, by_rate, rows_held) { - (true, by_rate, _) => by_rate.ok_or_else(|| missing_rate(source))?, - (false, Some(by_rate), Some(rows_held)) => by_rate.min(rows_held), - (false, by_rate, rows_held) => by_rate.or(rows_held).ok_or_else(|| { - AnalyticalCostError::MissingSourceFact { - source_name: source.to_string(), - missing: match lookback { - Some(_) => { - "an arrival rate (the arrival's rows_per_second, or the \ - source's series_count with sample_interval) or its \ - rows_held" - } - None => "its rows_held (the scan has no window, so it reads every row held)", - }, - } - })?, - }; - let out = edge(rows.max(1))?; - let estimate = estimate_operator( - PhysicalOperator::Scan, - OperatorStatistics::Scan { - edges: UnaryEdgeStatistics { - input: out, - output: out, - promql: None, - }, - source_read_bytes: out.bytes, - }, - ); - (out, estimate, format!("scan {} samples", out.rows)) - } - NonASAPOp::Aggregate { - reduction, - measures, - .. - } => { - let group_count = groups(reduction)?; - let keys = match reduction { - Reduction::Reduce(keys) => keys.keys().len() as u64, - _ => 1, - }; - let out = edge(group_count)?; - let held = input.rows.div_ceil(group_count.max(1)); - let estimate = estimate_operator( - PhysicalOperator::HashAggregate { - grouping_key_count: keys, - accumulator_count: measures.len().max(1) as u64, - }, - OperatorStatistics::HashAggregate { - edges: unary(out), - group_count, - key_bytes: keys.checked_mul(16).ok_or(AnalyticalCostError::Overflow)?, - accumulator_bytes_per_group: measures - .iter() - .try_fold(0_u64, |total, measure| { - let values = if keeps_every_value(measure) { held } else { 1 }; - total - .checked_add( - values - .checked_mul(8) - .ok_or(AnalyticalCostError::Overflow)?, - ) - .ok_or(AnalyticalCostError::Overflow) - })? - .max(8), - }, - ); - ( - out, - estimate, - format!( - "hash aggregate {} rows into {group_count} groups", - input.rows - ), - ) - } - NonASAPOp::Sort { - keys, partition_by, .. - } => { - let partitions = if partition_by.keys().is_empty() { - 1 - } else { - estimated_groups(partition_by)? - }; - let estimate = estimate_operator( - PhysicalOperator::InMemoryComparisonSort { - ordering_key_count: keys.len() as u64, - partitioned: partitions > 1, - }, - OperatorStatistics::InMemoryComparisonSort { - edges: unary(input), - input_partitioning: split(input, partitions), - }, - ); - ( - input, - estimate, - format!("sort {} rows in {partitions} partitions", input.rows), - ) - } - NonASAPOp::Limit { - n, - offset, - partition_by, - .. - } => { - let partitions = if partition_by.keys().is_empty() { - 1 - } else { - estimated_groups(partition_by)? - }; - let limit = n.map_or(u64::MAX, |n| (n as u64).saturating_mul(partitions)); - let offset = (*offset as u64).saturating_mul(partitions); - let out = edge(input.rows.saturating_sub(offset).min(limit))?; - let estimate = estimate_operator( - PhysicalOperator::Limit { limit, offset }, - OperatorStatistics::Limit { edges: unary(out) }, - ); - (out, estimate, format!("limit to {} rows", out.rows)) - } - NonASAPOp::TimeShift { .. } => ( - input, - estimate_operator( - PhysicalOperator::TimeShift, - OperatorStatistics::PassThrough { - edges: unary(input), - }, - ), - format!("time shift: pass {} rows, free", input.rows), - ), - NonASAPOp::TimeRange { range, kind, .. } => { - // A range keeps its window's samples; an instant - // selector the latest sample of each series. - // Unknown rate or series: no fewer than its input. - let kept = match kind { - TimeRangeKind::Range => rows_per_ms.map_or(input.rows, |rows_per_ms| { - (rows_per_ms * range.as_millis() as f64).round() as u64 - }), - TimeRangeKind::Instant => series.unwrap_or(input.rows), - }; - let out = edge(input.rows.min(kept.max(1)))?; - let out = EdgeStatistics { - rows: out.rows, - bytes: out.bytes.min(input.bytes), - }; - let estimate = estimate_operator( - PhysicalOperator::TimeRange, - OperatorStatistics::PassThrough { edges: unary(out) }, - ); - ( - out, - estimate, - format!( - "time range {range:?}: keep {} of {} rows", - out.rows, input.rows - ), - ) - } - _ => { - let estimate = estimate_operator( - PhysicalOperator::PassThrough, - OperatorStatistics::PassThrough { - edges: unary(input), - }, - ); - ( - input, - estimate, - format!("operator: pass {} rows", input.rows), - ) - } - }, - Payload::ASAP(ASAPOp::SummaryAgg { - family, reduction, .. - }) => { - let group_count = groups(reduction)?; - // At query time, each state sees its group's input rows; at - // ingestion time, the rows of a window one second of - // arriving rows does not bound. - let input_rows = StateInputRows { - rows_per_state: (!ingestion).then(|| input.rows.div_ceil(group_count.max(1))), - non_negative: raw_data_samples_non_negative(), - }; - let (depth, state_bytes) = summary_shape(family, input_rows); - let state_bytes = measured_state_bytes.unwrap_or(state_bytes); - let out = EdgeStatistics { - rows: group_count, - bytes: group_count - .checked_mul(state_bytes) - .ok_or(AnalyticalCostError::Overflow)?, - }; - ( - out, - Ok(ResourceEstimate::new( - input.rows as f64 * depth, - out.bytes, - 0, - )), - format!( - "build {} into {group_count} states: {} rows x depth {depth}", - family_name(family), - input.rows - ), - ) - } - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - reduction, - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => { - // Each input row is kept by the one tumbling window its time falls in, - // as a time range keeps it (one operation), and updates - // that tumbling window. - let group_count = groups(reduction)?; - let states = group_count - .checked_mul(u64::from(*tumbling_window_count)) - .ok_or(AnalyticalCostError::Overflow)?; - // Each tumbling window's state sees its group's rows in one tumbling window: of - // one second's rows at ingestion time, of the rows read at - // query time (all of them for a chain kept across - // evaluations, whose scan Stage 2 bounds to its newest tumbling window, - // the only one it builds). - let rows_per_tumbling_window = match (ingestion, node.materialization) { - (true, _) => u64::try_from( - (u128::from(input.rows) * u128::from(*tumbling_window_width_ms)) - .div_ceil(1_000), - ) - .map_err(|_| AnalyticalCostError::Overflow)?, - (false, Materialization::MatAtQueryKeptAcrossEvaluations(_)) => input.rows, - (false, _) => input - .rows - .div_ceil(u64::from(*tumbling_window_count).max(1)), - }; - let input_rows = StateInputRows { - rows_per_state: Some(rows_per_tumbling_window.div_ceil(group_count.max(1))), - non_negative: raw_data_samples_non_negative(), - }; - let (depth, state_bytes) = summary_shape(family, input_rows); - let state_bytes = measured_state_bytes.unwrap_or(state_bytes); - let out = EdgeStatistics { - rows: states, - bytes: states - .checked_mul(state_bytes) - .ok_or(AnalyticalCostError::Overflow)?, - }; - ( - out, - Ok(ResourceEstimate::new( - input.rows as f64 * (depth + 1.0), - out.bytes, - 0, - )), - format!( - "build {} in {tumbling_window_count} tumbling windows of {tumbling_window_width_ms} ms for \ - {group_count} groups: {} rows x (1 + depth {depth})", - family_name(family), - input.rows - ), - ) - } - Payload::ASAP(ASAPOp::SummaryMerge { - newest_tumbling_windows, - group_by, - .. - }) => { - // One operation per input state read; one state per group - // out, each tumbling window of a chain being a state of its group. A - // merge of a chain's newest `n` tumbling windows reads `n` per group. A - // rollup's groups are those of its keys, at most its input - // groups. - let mut group_count = 0; - for producer in producers { - let tumbling_windows = match producer.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => u64::from(*tumbling_window_count), - _ => 1, - }; - group_count = group_count.max(producer.output.rows / tumbling_windows.max(1)); - } - let states = merged_states(*newest_tumbling_windows, producers)?; - if let Some(keys) = group_by { - group_count = group_count.min(groups(&Reduction::by(keys.clone()))?); - } - let out = EdgeStatistics { - rows: group_count, - bytes: input - .bytes - .div_ceil(input.rows.max(1)) - .checked_mul(group_count) - .ok_or(AnalyticalCostError::Overflow)?, - }; - ( - out, - Ok(ResourceEstimate::new(states as f64, out.bytes, 0)), - format!("merge {states} states into {group_count}"), - ) - } - Payload::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { - let per_group = match query { - SketchStatistic::TopK { k } => *k as u64, - _ => 1, - }; - let out = edge( - input - .rows - .checked_mul(per_group) - .ok_or(AnalyticalCostError::Overflow)?, - )?; - ( - out, - Ok(ResourceEstimate::new(out.rows as f64, 0, 0)), - format!("estimate {} groups x {per_group}", input.rows), - ) - } - Payload::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) => ( - edge(input.rows)?, - Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), - format!("finalize {} accumulators", input.rows), - ), - _ => ( - edge(input.rows)?, - Ok(ResourceEstimate::new(input.rows as f64, 0, 0)), - format!("{} rows", input.rows), - ), - }; - let estimate = estimate?; - let (multiplier, per_second) = rate; - let formula_ms = estimate.cpu_ops() * speed.ms_per_cpu_op - + estimate.scan_bytes() as f64 * speed.ms_per_scanned_byte; - // What one computation processes, the unit its entry is priced per - // ([`OperatorKind`]): the samples a scan reads, the states a merge - // reads, otherwise the rows of its (first) input. - let units = match &node.payload { - Payload::NonASAP(NonASAPOp::Scan { .. }) => out.rows, - Payload::ASAP(ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - }) => merged_states(*newest_tumbling_windows, producers)?, - _ => input.rows, - }; - let mut detail = detail; - // A node with an entry takes its measured time, scanning included; - // any other node its formulas' CPU operations times the calibration - // factor, its scanned bytes unscaled (scaling only the CPU keeps it - // non-negative whatever the factor). - let estimate = match unit_price { - Some(unit) => { - price_source = NodePriceSource::Measured; - let measured_ms = (units as f64 * unit.ns_per_unit + unit.ns_per_invocation) * 1e-6; - fit.record(measured_ms * per_second, formula_ms * per_second); - detail += &format!( - "; measured {units} x {:.3} ns + {:.3} ns", - unit.ns_per_unit, unit.ns_per_invocation - ); - ResourceEstimate::new( - measured_ms / speed.ms_per_cpu_op, - estimate.peak_memory_bytes(), - 0, - ) - } - None if formula_scale != 1.0 => { - detail += &format!("; formula x {formula_scale:.3} (calibration)"); - ResourceEstimate::new( - estimate.cpu_ops() * formula_scale, - estimate.peak_memory_bytes(), - estimate.scan_bytes(), - ) - } - None => estimate, - }; - let node_ms = estimate.cpu_ops() * speed.ms_per_cpu_op - + estimate.scan_bytes() as f64 * speed.ms_per_scanned_byte; - - // What this node keeps: memory averaged over time (byte-seconds per - // second), its peak memory and its disk bytes, added to the - // candidate's usage below. - let mut average_memory_bytes = 0.0; - let mut peak_memory_bytes = 0u64; - let mut disk_bytes = 0u64; - let mut detail = format!("{detail}; x {per_second:.4}/s"); - // State kept between computations: at ingestion time, the window - // being built and the completed ones queries read (a chain of N - // tumbling windows keeps N + 1), every second. At query time the output is - // reused only within one evaluation, so it is held only while that - // evaluation runs (counted below, once latencies are known). The - // state is what the node holds: its output, or the values an exact - // aggregate keeps per group when that is more. A computation that - // keeps nothing holds its working memory while it runs, once per - // consumer that recomputes it; at query time, that is held for the - // evaluation too (counted in [`plan_cost`], per consumer: a query's - // latency, or a materialized consumer's longest reader's, as a - // MatAtQuery state is). - let state = out.bytes.max(estimate.peak_memory_bytes()); - let (kept, medium) = match (node.materialization, &node.payload) { - (Materialization::NotMaterialized, _) => { - let working_memory_bytes = estimate.peak_memory_bytes(); - peak_memory_bytes = working_memory_bytes.saturating_mul(multiplier); - if !ingestion && working_memory_bytes > 0 { - held = Some((working_memory_bytes, per_second, true)); - } - (0, Medium::Memory) - } - // A tumbling-window chain kept across evaluations holds the tumbling windows its - // retention covers ([`tumbling_window_retention_ms`]: N, plus one per tumbling window - // width of the readers' range-evaluation span), all but the - // newest between evaluations and all while one runs. - (Materialization::MatAtQueryKeptAcrossEvaluations(medium), _) => { - let tumbling_windows = tumbling_windows_of(node); - let retained = retained_tumbling_window_count; - let between = (state / tumbling_windows) - .checked_mul( - retained - .checked_sub(1) - .ok_or(AnalyticalCostError::Overflow)?, - ) - .ok_or(AnalyticalCostError::Overflow)?; - match medium { - Medium::Memory => { - average_memory_bytes = between as f64; - peak_memory_bytes = match retained == tumbling_windows { - true => state, - false => (state / tumbling_windows) - .checked_mul(retained) - .ok_or(AnalyticalCostError::Overflow)?, - }; - } - Medium::Disk => disk_bytes = between, - } - detail += &format!("; keeps {between} bytes across evaluations"); - (0, medium) - } - (Materialization::MatAtQuery(medium), _) => { - match medium { - Medium::Memory => { - held = Some((state, per_second, false)); - peak_memory_bytes = state; - } - Medium::Disk => disk_bytes = state, - } - (0, medium) - } - // The retained tumbling windows and the one being built. - ( - Materialization::MatAtIngestion(medium), - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - ) => { - let tumbling_windows = tumbling_windows_of(node); - ( - (state / tumbling_windows) - .checked_mul( - retained_tumbling_window_count - .checked_add(1) - .ok_or(AnalyticalCostError::Overflow)?, - ) - .ok_or(AnalyticalCostError::Overflow)?, - medium, - ) - } - (Materialization::MatAtIngestion(medium), _) => ( - state.checked_mul(2).ok_or(AnalyticalCostError::Overflow)?, - medium, - ), - }; - if node.materialization == Materialization::MatAtIngestion(Medium::Memory) { - maintained = Some(state / tumbling_windows_of(node)); - } - if kept > 0 { - match medium { - Medium::Memory => { - average_memory_bytes = kept as f64; - peak_memory_bytes = kept; - } - Medium::Disk => disk_bytes = kept, - } - detail += &format!("; keeps {kept} bytes"); - } - if let Payload::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload { - if !ingestion { - let kept = match raw_kept_rows { - Some(rows) => edge(rows)?.bytes, - None => out.bytes, - }; - raw_scan = Some((source, kept)); - } - } - // One computation's CPU and scan price times its rate, then the - // kept state's. - let cost = usage_prices.map(|prices| { - (estimate.cpu_ops() * prices.cost_per_cpu_op - + estimate.scan_bytes() as f64 * prices.cost_per_scanned_byte) - * per_second - + prices.cost_per_memory_byte_second * average_memory_bytes - + prices.cost_per_disk_byte_second * disk_bytes as f64 - }); - if cost.is_some_and(|cost| !cost.is_finite()) { - return Err(AnalyticalCostError::Overflow); - } - Ok(NodeUsage { - output: out, - sources, - family, - unchanged_columns, - work_ms_per_execution: node_ms, - cpu_ops_per_second: estimate.cpu_ops() * per_second, - scanned_bytes_per_second: estimate.scan_bytes() as f64 * per_second, - average_memory_bytes, - peak_memory_bytes, - disk_bytes, - held, - maintained, - raw_scan, - fit, - cost: NodeCost { - operator, - timing: node.output_state.timing, - cost_per_second: cost, - cpu_core_seconds_per_second: node_ms * per_second / 1_000.0, - multiplier, - detail, - price_source, - }, - }) -} - -/// [`PlanCost::source`] for nodes priced as `per_node` says. -fn cost_source(per_node: &BTreeMap) -> &'static str { - let measured = |node: &NodeCost| node.price_source == NodePriceSource::Measured; - if !per_node.values().any(measured) { - COST_SOURCE - } else if per_node.values().all(measured) { - MEASURED_COST_SOURCE - } else { - MIXED_COST_SOURCE - } -} - -/// The CPU cores a candidate needs at its peak: enough for its -/// ingestion-time work every second (none without any), plus one per -/// query with query-time work, all evaluating at once; at least the cores -/// its average query-time work fills. -fn peak_cpu_cores( - dag: &PhysicalASAPDAG, - nodes: &HashMap, - work_ms: &HashMap, - multipliers: &HashMap, - latency: &[QueryLatency], - request_ms_per_second: f64, -) -> u32 { - let mut ingestion_ms_per_second = 0.0; - // Request overhead is attributed in per_node reports, not work_ms, and so - // must enter the throughput floor separately (including literal queries). - let mut query_ms_per_second = request_ms_per_second; - for node in &dag.nodes { - let per_second = multipliers[&node.id].1; - if nodes[&node.id].output_state.timing == ExecutionTiming::IngestionTime { - ingestion_ms_per_second += work_ms[&node.id] * per_second; - } else { - query_ms_per_second += work_ms[&node.id] * per_second; - } - } - let cores = |ms_per_second: f64| (ms_per_second / 1_000.0).ceil() as u32; - let evaluating = latency.iter().filter(|l| l.query_ms > 0.0).count() as u32; - cores(ingestion_ms_per_second) + evaluating.max(cores(query_ms_per_second)) -} - -/// Each query's latency per evaluation (module doc, "Latency"): the sum of -/// the query-time work it reaches, each node once, plus the longest -/// compaction among the ingestion-time states it reads. Compacting a -/// state's newest window merges, for each of its `I` states per window, -/// the `k` ingest workers' partial copies, `(k − 1)·I` merges, where `k` -/// is the cores its ingestion work takes, at least one. Each merge takes -/// the state's measured merge time (`merge_ms`) when the measured table has -/// one, and one operation otherwise. -fn query_latencies( - dag: &PhysicalASAPDAG, - nodes: &HashMap, - output: &HashMap, - work_ms: &HashMap, - merge_ms: &HashMap, - formula_merge_ms: f64, -) -> Vec { - let inputs = |id: PhysicalASAPNodeId| { - dag.edges - .iter() - .filter(move |e| e.consumer == id) - .map(|e| e.producer) - }; - let at_ingestion = - |id: PhysicalASAPNodeId| nodes[&id].output_state.timing == ExecutionTiming::IngestionTime; - let compaction_ms = |state: PhysicalASAPNodeId| { - let workers = ingest_workers(dag, work_ms, state) as f64; - let states = (output[&state].rows / tumbling_windows_of(nodes[&state])) as f64; - let per_merge_ms = merge_ms.get(&state).copied().unwrap_or(formula_merge_ms); - (workers - 1.0) * states * per_merge_ms - }; - query_starts(dag) - .into_iter() - .map(|start| { - let mut latency = QueryLatency { - compaction_ms: 0.0, - query_ms: 0.0, - }; - let mut seen = HashSet::new(); - let mut stack = start; - while let Some(id) = stack.pop() { - if !seen.insert(id) { - continue; - } - if at_ingestion(id) { - latency.compaction_ms = latency.compaction_ms.max(compaction_ms(id)); - continue; - } - latency.query_ms += work_ms[&id]; - stack.extend(inputs(id)); - } - latency - }) - .collect() -} - -/// The cores `state`'s ingestion work takes, at least one: the ingest -/// workers that each build a partial copy of its newest window. -fn ingest_workers( - dag: &PhysicalASAPDAG, - work_ms: &HashMap, - state: PhysicalASAPNodeId, -) -> u64 { - let mut ingest_ms_per_s = 0.0; - let mut seen = HashSet::new(); - let mut stack = vec![state]; - while let Some(id) = stack.pop() { - if seen.insert(id) { - ingest_ms_per_s += work_ms[&id]; - stack.extend( - dag.edges - .iter() - .filter(|e| e.consumer == id) - .map(|e| e.producer), - ); - } - } - (ingest_ms_per_s / 1_000.0).ceil().max(1.0) as u64 -} - -/// For each tumbling-window chain of `dag` kept beyond one evaluation (at -/// ingestion time, or at query time across evaluations), how far back -/// from the newest evaluation its completed tumbling windows must be kept, in -/// milliseconds: how long before the evaluation its newest tumbling window ends (the -/// offset of its input, `chain_input_offset_ms`), plus its window -/// `tumbling_window_count · tumbling_window_width_ms`, plus the longest -/// `span_ms` ([`RangeEvaluation`]) among the queries reaching it, -/// whose earliest step reads that much further back. `reach` is -/// [`query_reach`] of `dag`; `range_evaluations[q]` is query `q`'s (none -/// when missing). Without range evaluations, each chain's window. -pub fn tumbling_window_retention_ms( - dag: &PhysicalASAPDAG, - reach: &[HashSet], - range_evaluations: &[Option], -) -> BTreeMap { - let nodes = node_map(dag); - dag.nodes - .iter() - .filter_map(|node| match (&node.payload, node.materialization) { - ( - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - child, - tumbling_window_width_ms, - tumbling_window_count, - .. - }), - Materialization::MatAtIngestion(_) - | Materialization::MatAtQueryKeptAcrossEvaluations(_), - ) => { - let span_ms = reach - .iter() - .enumerate() - .filter(|(_, reached)| reached.contains(&node.id)) - .filter_map(|(q, _)| range_evaluations.get(q).copied().flatten()) - .map(|range| range.span_ms) - .max() - .unwrap_or(0); - let end_offset_ms = chain_input_offset_ms(*child, &nodes); - Some(( - node.id, - end_offset_ms - + tumbling_window_width_ms * u64::from(*tumbling_window_count) - + span_ms, - )) - } - _ => None, - }) - .collect() -} - -/// How long before the evaluation a chain's newest tumbling window ends: the offsets -/// of the `TimeShift`s (without an `@` anchor) directly below the chain, -/// added up; 0 for a negative total. -fn chain_input_offset_ms( - input: PhysicalASAPNodeId, - nodes: &HashMap, -) -> u64 { - let mut offset_ms: i64 = 0; - let mut current = input; - while let Some(Payload::NonASAP(NonASAPOp::TimeShift { shift, child })) = - nodes.get(¤t).map(|node| &node.payload) - { - if shift.at.is_some() { - break; - } - offset_ms = offset_ms.saturating_add(shift.offset_ms); - current = *child; - } - u64::try_from(offset_ms).unwrap_or(0) -} - -/// The tumbling-window retention of every kept tumbling-window chain of `dag` -/// ([`tumbling_window_retention_ms`] over [`query_reach`]), for a plan whose query -/// `q` has range evaluation `range_evaluations[q]` (none when missing): -/// what Stage 2's realizer records for a realized plan. -pub fn tumbling_window_retention_ms_of_plan( - dag: &PhysicalASAPDAG, - range_evaluations: &[Option], -) -> BTreeMap { - tumbling_window_retention_ms(dag, &query_reach(dag, &node_map(dag)), range_evaluations) -} - -/// The input states a `SummaryMerge` over `producers` reads: every row of -/// each, or of the one tumbling-window chain whose newest `newest_tumbling_windows` it merges, -/// that many tumbling windows' states per group. -fn merged_states( - newest_tumbling_windows: Option, - producers: &[ProducerOutput<'_>], -) -> Result { - producers.iter().try_fold(0_u64, |total, producer| { - let rows = match (newest_tumbling_windows, producer.payload) { - ( - Some(count), - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }), - ) => (producer.output.rows / u64::from(*tumbling_window_count).max(1)) - .checked_mul(u64::from(count)) - .ok_or(AnalyticalCostError::Overflow)?, - _ => producer.output.rows, - }; - total.checked_add(rows).ok_or(AnalyticalCostError::Overflow) - }) -} - -/// The panes a chain in `retention_ms` ([`tumbling_window_retention_ms`]) keeps -/// completed: its retention over its pane width, at least its pane count. -pub fn retained_tumbling_windows( - node: &asap_types::ir::physical_export::PhysicalASAPDAGNode, - retention_ms: &BTreeMap, -) -> u64 { - match (&node.payload, retention_ms.get(&node.id)) { - ( - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }), - Some(&retention), - ) => (retention / tumbling_window_width_ms.max(&1)).max(tumbling_windows_of(node)), - _ => tumbling_windows_of(node), - } -} - -/// A node's tumbling windows: a tumbling-window chain's count, 1 for any other node. -pub fn tumbling_windows_of(node: &asap_types::ir::physical_export::PhysicalASAPDAGNode) -> u64 { - match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => u64::from(*tumbling_window_count).max(1), - _ => 1, - } -} - -/// The nodes each query reaches at query time, in workload order: its walk -/// from its roots, stopping at (and including) ingestion-time nodes. -pub fn query_reach( - dag: &PhysicalASAPDAG, - nodes: &HashMap, -) -> Vec> { - query_starts(dag) - .into_iter() - .map(|mut stack| { - let mut seen = HashSet::new(); - while let Some(id) = stack.pop() { - if seen.insert(id) - && nodes[&id].output_state.timing != ExecutionTiming::IngestionTime - { - stack.extend( - dag.edges - .iter() - .filter(|e| e.consumer == id) - .map(|e| e.producer), - ); - } - } - seen - }) - .collect() -} - -/// The window an ingestion-time grouping `node` maintains: a tumbling-window chain's -/// tumbling windows, or the time range below it (through filters, projections and -/// time shifts). `None` when there is none. -pub fn maintained_window_ms( - node: PhysicalASAPNodeId, - nodes: &NodeMap<'_>, - producers_of: &HashMap>, -) -> Option { - match &nodes[&node].payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => { - return Some(tumbling_window_width_ms.saturating_mul(u64::from(*tumbling_window_count))) - } - Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) => { - return Some(range.as_millis() as u64) - } - _ => {} - } - let producer = *producers_of.get(&node)?.first()?; - match &nodes[&producer].payload { - Payload::NonASAP( - NonASAPOp::TimeRange { .. } - | NonASAPOp::Filter { .. } - | NonASAPOp::Project { .. } - | NonASAPOp::TimeShift { .. }, - ) => maintained_window_ms(producer, nodes, producers_of), - _ => None, - } -} - -/// The window of a scan reading the `lookback_ms` up to the evaluation -/// (`Unbounded` for `None`). -pub fn trailing_window(lookback_ms: Option) -> ScanWindow { - lookback_ms.map_or(ScanWindow::Unbounded, |width_ms| ScanWindow::Within { - end_offset_ms: 0, - width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), - }) -} - -/// The nodes of `dag` by id. -pub fn node_map(dag: &PhysicalASAPDAG) -> NodeMap<'_> { - dag.nodes.iter().map(|n| (n.id, n)).collect() -} - -/// The nodes of a physical DAG by id. -pub type NodeMap<'a> = - HashMap; - -/// The tumbling-window chains of `dag` kept across evaluations -/// (`MatAtQueryKeptAcrossEvaluations`, which Stage 2 offers only when the -/// deployment keeps query-time results and every reader builds one new -/// tumbling window per evaluation). Each evaluation builds the newest tumbling window, and the -/// older ones, which complete the readers' look-back, are kept from -/// earlier evaluations. -pub fn kept_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet { - dag.nodes - .iter() - .filter(|node| { - matches!( - (&node.payload, node.materialization), - ( - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - Materialization::MatAtQueryKeptAcrossEvaluations(_), - ) - ) - }) - .map(|node| node.id) - .collect() -} - -/// The widest window, in milliseconds, read above the scan `scan` at query -/// time, ignoring readers that read it outside a window and the anchors of -/// shifts: each range `TimeRange` and `SummaryTumblingWindowChain` (its whole window, or -/// one tumbling window for a chain in `kept_chains`) reached through time shifts. -/// Prices a scan whose [`PhysicalASAPDAG::scan_window`] is unbounded as -/// Stage 3 did before; #955 decides `@` anchors. -fn widest_window_read_ms( - dag: &PhysicalASAPDAG, - nodes: &NodeMap<'_>, - scan: PhysicalASAPNodeId, - kept_chains: &HashSet, -) -> Option { - let mut widest = None; - let mut stack = vec![scan]; - let mut seen = HashSet::new(); - while let Some(id) = stack.pop() { - if !seen.insert(id) { - continue; - } - for edge in dag.edges.iter().filter(|e| e.producer == id) { - let width_ms = match &nodes[&edge.consumer].payload { - Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) => range.as_millis() as u64, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) if kept_chains.contains(&edge.consumer) => *tumbling_window_width_ms, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => tumbling_window_width_ms.saturating_mul(u64::from(*tumbling_window_count)), - Payload::NonASAP(NonASAPOp::TimeShift { .. }) => { - stack.push(edge.consumer); - continue; - } - _ => continue, - }; - widest = widest.max(Some(width_ms)); - } - } - widest -} - -/// A summary family's name in reports: a shared (Hydra) structure by its -/// Hydra kind, e.g. `HydraCms`; a sketch by its algorithm; an exact -/// accumulator by its kind. -pub fn family_name(family: &FieldDataType) -> String { - match family { - FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { - format!("{kind:?}") - } - FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), - FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), - other => format!("{other:?}"), - } -} - -/// `input` split as evenly as integers allow into `partitions` parts. -fn split(input: EdgeStatistics, partitions: u64) -> PartitionStatistics { - let part = |total: u64, i: u64| total / partitions + u64::from(i < total % partitions); - PartitionStatistics { - partitions: (0..partitions) - .map(|i| EdgeStatistics { - rows: part(input.rows, i), - bytes: part(input.bytes, i), - }) - .collect(), - } -} diff --git a/crates/cost/tests/held_memory.rs b/crates/cost/tests/held_memory.rs deleted file mode 100644 index 8bc3978c0..000000000 --- a/crates/cost/tests/held_memory.rs +++ /dev/null @@ -1,168 +0,0 @@ -//! Query-time working memory in [`plan_cost`]: how long a `NotMaterialized` -//! node's working memory is held depends on whether a materialized node -//! reads it. - -use asap_cost::execution_demand::evaluation_rate; -use asap_cost::ResourceUnitCost; -use asap_cost::{ - kept_tumbling_window_chains, node_map, plan_cost, CostInputs, CostModel, PlanCost, -}; -use asap_logical_asap_optimization::test_support::lower_promql; -use asap_physical_asap_optimization::optimize_physical_asap; -use asap_types::ir::physical_export::PhysicalASAPDAG; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::QueryRoot; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{ - DataArrival, DataWorkload, QueryRecurrence, RepeatedDemand, RepetitionInterval, SourceFacts, -}; - -/// Data arriving at 10 000 rows per second over 10 000 series of `m`. -fn data() -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 10_000.0, - }, - sources: [( - "m".to_string(), - SourceFacts { - series_count: Some(10_000), - ..SourceFacts::default() - }, - )] - .into_iter() - .collect(), - } -} - -/// Two queries repeating every 10 s and every 20 s. -fn recurrence() -> [QueryRecurrence; 2] { - [10_000, 20_000].map(|interval_ms| { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( - interval_ms, - ))) - }) -} - -/// `plan_cost` of `dag` for [`recurrence`], at `cost_per_memory_byte_second`. -fn cost(dag: &PhysicalASAPDAG, cost_per_memory_byte_second: f64) -> PlanCost { - let mut unit_cost = ResourceUnitCost::illustrative(); - unit_cost.cost_per_memory_byte_second = cost_per_memory_byte_second; - let settings = CostInputs { - cost_model: CostModel::UsageOverTime(unit_cost), - ..CostInputs::default() - }; - let nodes = node_map(dag); - plan_cost( - dag, - &nodes, - &kept_tumbling_window_chains(dag), - &recurrence(), - &[], - &data(), - &settings, - 1.0, - ) - .unwrap() - .0 -} - -/// Per node holding query-time memory, (its id, its held bytes, its -/// held-memory cost per second at unit memory cost 1). -fn held_memory_costs(dag: &PhysicalASAPDAG) -> Vec<(usize, f64, f64)> { - let (with, without) = (cost(dag, 1.0), cost(dag, 0.0)); - with.per_node - .iter() - .filter_map(|(id, node)| { - let held = node.detail.split("; holds ").nth(1)?; - let bytes: f64 = held.split(' ').next().unwrap().parse().unwrap(); - let term = - node.cost_per_second.unwrap() - without.per_node[id].cost_per_second.unwrap(); - Some((*id, bytes, term)) - }) - .collect() -} - -fn materialization_of(dag: &PhysicalASAPDAG, id: usize) -> Materialization { - dag.nodes - .iter() - .find(|n| n.id == id) - .unwrap() - .materialization -} - -fn close(a: f64, b: f64) -> bool { - (a - b).abs() <= 1e-12 * a.abs().max(b.abs()).max(1e-300) -} - -/// Query 0 is `sum(...)` over query 1's output, so it runs longer. Without -/// a materialized node, each query computes the shared nodes and holds -/// their working memory for its own latency: `bytes · Σ rate_q · latency_q`. -/// When query 1's output is `MatAtQuery`, the nodes below it are computed -/// once per execution of it, at its rate, and held until the -/// longest reader, query 0, finishes, as the `MatAtQuery` state itself is: -/// `bytes · rate · latency_0`, not the rate-weighted mean of the two -/// latencies. -#[test] -fn working_memory_read_by_a_materialized_node_is_held_for_the_longest_reader() { - let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::exact()); - let inner = outer.children()[0].clone(); - let plans = optimize_physical_asap( - "L", - &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], - &[], - &data(), - false, - ) - .unwrap(); - let dag_with = |materialization: Materialization| { - plans - .iter() - .map(|plan| &plan.dag) - .find(|dag| materialization_of(dag, dag.roots[1]) == materialization) - .unwrap() - }; - let recomputed = dag_with(Materialization::NotMaterialized); - let shared = dag_with(Materialization::MatAtQuery(Medium::Memory)); - let latency_ms = |dag| { - let cost = cost(dag, 1.0); - [0, 1].map(|q| cost.usage.latency_per_query[q].total_ms()) - }; - - // Recomputed per query: the rate-weighted sum of both latencies. - let [outer_ms, inner_ms] = latency_ms(recomputed); - assert!(outer_ms > inner_ms, "{outer_ms} vs {inner_ms}"); - let below = |dag: &PhysicalASAPDAG| { - let materialized = dag.roots[1]; - held_memory_costs(dag) - .into_iter() - .filter(move |(id, _, _)| *id != materialized && *id != dag.roots[0]) - .collect::>() - }; - let checked = below(recomputed); - assert!(!checked.is_empty(), "the shared nodes hold memory"); - for (id, bytes, term) in checked { - let expected = bytes * (0.1 * outer_ms + 0.05 * inner_ms) / 1_000.0; - assert!(close(term, expected), "node {id}: {term} vs {expected}"); - } - - // Below a MatAtQuery node: the longest reader's latency at the - // materialized node's rate (its readers' schedules together). - let materialized_rate = evaluation_rate( - recurrence().iter().map(|r| (r, None)), - CostInputs::default().non_repeating_query_horizon_seconds, - ) - .unwrap(); - let [outer_ms, inner_ms] = latency_ms(shared); - assert!(outer_ms > inner_ms, "{outer_ms} vs {inner_ms}"); - let checked = below(shared); - assert!(!checked.is_empty(), "the nodes below it hold memory"); - for (id, bytes, term) in checked { - assert_eq!( - materialization_of(shared, id), - Materialization::NotMaterialized - ); - let expected = bytes * materialized_rate * outer_ms / 1_000.0; - assert!(close(term, expected), "node {id}: {term} vs {expected}"); - } -} diff --git a/crates/cost/tests/ingestion_refresh_runs.rs b/crates/cost/tests/ingestion_refresh_runs.rs deleted file mode 100644 index ebd33bab5..000000000 --- a/crates/cost/tests/ingestion_refresh_runs.rs +++ /dev/null @@ -1,289 +0,0 @@ -//! How often a `NotMaterialized` node is computed for the views refreshed -//! at ingestion above it ([`computations`]): once per refresh window, since -//! a runtime refreshes the views of one refresh window in one ingestion run -//! (issue #1151). - -use std::rc::Rc; -use std::time::Duration; - -use asap_cost::execution_demand::computations; -use asap_types::ir::operator::operator_properties::{Reduction, Source, TimeShift}; -use asap_types::ir::physical_export::{compile_physical_asap_workload, PhysicalASAPDAG}; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchKind, - SketchParams, SummaryUpdate, -}; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, - OperatorNode, QueryRoot, TimeRangeKind, -}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - -/// `m` with columns `ts`, `value` and `job`. -fn scan() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -/// A quantile sketch of `algorithm` (KLL k = 200 or DDSketch α = 0.01). -fn quantile_sketch(algorithm: SketchAlgorithm) -> FieldDataType { - let params = match algorithm { - SketchAlgorithm::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - _ => SketchParams::DDSketch { alpha: 0.01 }, - }; - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) -} - -/// `tumbling_window_count` tumbling windows of `tumbling_window_width_ms` of a quantile sketch of -/// `algorithm` per `job` over `input`. -fn chain( - input: &Rc, - algorithm: SketchAlgorithm, - tumbling_window_width_ms: u64, - tumbling_window_count: u32, -) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(input), - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - tumbling_window_width_ms, - tumbling_window_count, - })) - .unwrap() -} - -/// One query per summary of `summaries`, merging it, with each summary -/// materialized as `materialization`: the DAG Stage 3 costs, chains -/// unexpanded. -fn dag_of(summaries: &[Rc], materialization: Materialization) -> PhysicalASAPDAG { - let mut assignment = MaterializationAssignment::default(); - let roots: Vec = summaries - .iter() - .map(|summary| { - if materialization != Materialization::NotMaterialized { - assignment.set(summary, materialization); - } - QueryRoot::Operator( - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(summary)], - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(), - ) - }) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - compile_physical_asap_workload(&timed).unwrap() -} - -/// `(multiplier, per_second)` of `dag`'s `Scan`, each query every 10 s. -fn scan_computations(dag: &PhysicalASAPDAG) -> (u64, f64) { - computations_of(dag, |op| matches!(op, NonASAPOp::Scan { .. })) -} - -/// `(multiplier, per_second)` of the one node of `dag` whose operator -/// `is_node` picks, each query every 10 s. -fn computations_of( - dag: &PhysicalASAPDAG, - is_node: impl Fn(&NonASAPOp) -> bool, -) -> (u64, f64) { - let every_10_s = - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(10_000))); - let recurrence = vec![every_10_s; dag.roots.len()]; - let times = computations(dag, &recurrence, &[], 3600.0, None).unwrap(); - let node = dag - .nodes - .iter() - .find(|node| matches!(&node.payload, Operator::NonASAP(op) if is_node(op))) - .unwrap(); - times[&node.id] -} - -/// `child` evaluated `offset_ms` earlier. -fn shifted(child: &Rc, offset_ms: i64) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms, - at: None, - }, - child: Rc::clone(child), - })) - .unwrap() -} - -const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - -/// A KLL chain and a DDSketch chain of 10 s tumbling windows over one `Scan`, both -/// kept at ingestion, are refreshed in one ingestion run per tumbling window: the -/// `Scan` is computed once per second of arriving data, not twice. -#[test] -fn a_scan_under_two_chains_of_one_tumbling_window_width_is_computed_once() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain(&scan, SketchAlgorithm::DDSketch, 10_000, 30), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (1, 1.0)); -} - -/// With 10 s and 60 s tumbling windows the chains need different runs (the 10 s chain -/// needs five tumbling window ends the 60 s chain does not): the `Scan` is computed -/// once for each. -#[test] -fn a_scan_under_chains_of_two_tumbling_window_widths_is_computed_for_each() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain(&scan, SketchAlgorithm::DDSketch, 60_000, 5), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (2, 2.0)); -} - -/// Query consumers are unchanged: with both chains built at query time -/// (`NotMaterialized`), each query computes the `Scan` at its own rate. -#[test] -fn query_consumers_still_count_once_each() { - let scan = scan(); - let range = |width_ms| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(width_ms), - kind: TimeRangeKind::Range, - child: Rc::clone(&scan), - })) - .unwrap() - }; - let build = |input: Rc, algorithm| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: input, - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() - }; - let dag = dag_of( - &[ - build(range(60_000), SketchAlgorithm::Kll), - build(range(60_000), SketchAlgorithm::DDSketch), - ], - Materialization::NotMaterialized, - ); - assert_eq!(scan_computations(&dag), (2, 0.2)); -} - -/// Two 10 s chains and a 60 s chain over one `Scan`: two refresh windows, -/// two ingestion runs. -#[test] -fn a_scan_under_chains_of_10_10_and_60_s_is_computed_for_each_width() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain(&scan, SketchAlgorithm::DDSketch, 10_000, 30), - chain(&scan, SketchAlgorithm::Kll, 60_000, 5), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (2, 2.0)); -} - -/// A 10 s chain over the `Scan` and a 10 s chain over the `Scan` shifted by -/// 5 s end their tumbling windows on different grids (phases 0 and 5 s): two runs. -#[test] -fn chains_of_one_width_and_two_phases_are_computed_apart() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain( - &shifted(&scan, 5_000), - SketchAlgorithm::DDSketch, - 10_000, - 30, - ), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (2, 2.0)); -} - -/// Two 10 s chains over one `TimeShift(10 s)` of the `Scan` (phase 0 for -/// both): the shift and the `Scan` below it run once per run. -#[test] -fn nodes_below_a_shared_intermediate_node_are_computed_once_per_run() { - let shift = shifted(&scan(), 10_000); - let dag = dag_of( - &[ - chain(&shift, SketchAlgorithm::Kll, 10_000, 6), - chain(&shift, SketchAlgorithm::DDSketch, 10_000, 30), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (1, 1.0)); - assert_eq!( - computations_of(&dag, |op| matches!(op, NonASAPOp::TimeShift { .. })), - (1, 1.0) - ); -} - -/// Two summaries kept at ingestion over rows with no time window -/// (`Values`) have no refresh window: each counts alone. -#[test] -fn consumers_of_unbounded_windows_count_alone() { - let rows = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![], - schema: scan().schema.clone(), - })) - .unwrap(); - let build = |algorithm| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(&rows), - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() - }; - let dag = dag_of( - &[ - build(SketchAlgorithm::Kll), - build(SketchAlgorithm::DDSketch), - ], - AT_INGESTION, - ); - assert_eq!( - computations_of(&dag, |op| matches!(op, NonASAPOp::Values { .. })), - (2, 2.0) - ); -} diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 763e654fe..bb24e585b 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -13,9 +13,9 @@ asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } asap-accuracy = { path = "../accuracy" } asap-logical-asap-optimization = { path = "../logical-asap-optimization" } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["reference-pricer"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } asap-cost = { path = "../cost" } -asap-plan-selection = { path = "../plan-selection", features = ["reference-pricer"] } +asap-plan-selection = { path = "../plan-selection" } # The installable plan of the selected candidate (`installable_plan_of`), # whose stored outputs `asap_planner` writes. asap-planner = { path = "../planner" } @@ -32,14 +32,9 @@ serde_json = "1" serde_yaml = "0.9" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } -[features] -# Each Stage 3 selection MILP solve also reports where its LP relaxation -# loses its bound, with `ASAP_STAGE3_LP_DIAGNOSIS=` (measurement only). -lp-diagnosis = ["asap-plan-selection/lp-diagnosis"] - [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } # Pass 1's target list (`per_computation_candidates`) for the tests. asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } diff --git a/crates/devtools/examples/canonical_examples.rs b/crates/devtools/examples/canonical_examples.rs index fbfb65be6..c618a2e26 100644 --- a/crates/devtools/examples/canonical_examples.rs +++ b/crates/devtools/examples/canonical_examples.rs @@ -6,7 +6,6 @@ use asap_devtools::corpora::{bgp_catalog, dqc_catalog}; use asap_devtools::lower_promql_with_sample_interval; use asap_frontend_sql::lower_sql_dialect; -use asap_types::ir::properties::AccuracyTarget; use asap_types::workload::SqlDialect; #[tokio::main] @@ -35,7 +34,7 @@ async fn main() { ]; for (label, q) in promql_examples { println!("=== {label} === promql> {q}"); - match lower_promql_with_sample_interval(q, AccuracyTarget::exact(), 1_000) { + match lower_promql_with_sample_interval(q, 1_000) { Ok(qe) => println!("{qe:#?}"), Err(e) => println!("ERR: {e}"), } @@ -63,14 +62,7 @@ async fn main() { &packets }; println!("=== {label} === sql> {q}"); - match lower_sql_dialect( - q, - catalog, - SqlDialect::DataFusionSQL, - AccuracyTarget::exact(), - ) - .await - { + match lower_sql_dialect(q, catalog, SqlDialect::DataFusionSQL).await { Ok(qe) => println!("{qe:#?}"), Err(e) => println!("ERR: {e}"), } diff --git a/crates/devtools/examples/topk_ir.rs b/crates/devtools/examples/topk_ir.rs index 5464fcae4..1765854de 100644 --- a/crates/devtools/examples/topk_ir.rs +++ b/crates/devtools/examples/topk_ir.rs @@ -4,7 +4,6 @@ // resulting pre-ASAP IR. Used for interactive exploration; not a test. use asap_devtools::{lower_promql_with_sample_interval, lower_sql, SqlCatalog}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; fn col(name: &str, dtype: DataType) -> Field { @@ -39,7 +38,7 @@ fn catalog() -> SqlCatalog { async fn show_sql(label: &str, query: &str) { println!("━━━ {label} ━━━"); println!("{query}"); - match lower_sql(query, &catalog(), AccuracyTarget::exact()).await { + match lower_sql(query, &catalog()).await { Ok(qe) => println!("{qe:#?}"), Err(e) => println!("ERR: {e}"), } @@ -49,7 +48,7 @@ async fn show_sql(label: &str, query: &str) { fn show_promql(label: &str, query: &str) { println!("━━━ {label} ━━━"); println!("{query}"); - match lower_promql_with_sample_interval(query, AccuracyTarget::exact(), 1_000) { + match lower_promql_with_sample_interval(query, 1_000) { Ok(qe) => println!("{qe:#?}"), Err(e) => println!("ERR: {e}"), } diff --git a/crates/devtools/src/bin/analyze_corpora.rs b/crates/devtools/src/bin/analyze_corpora.rs index 5841cfab9..8a50f6698 100644 --- a/crates/devtools/src/bin/analyze_corpora.rs +++ b/crates/devtools/src/bin/analyze_corpora.rs @@ -9,7 +9,6 @@ use asap_devtools::corpora::{ }; use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog}; use asap_frontend_sql::lower_sql_dialect; -use asap_types::ir::properties::AccuracyTarget; use asap_types::workload::SqlDialect; use serde::Serialize; use serde_json::Value; @@ -124,7 +123,7 @@ fn run_corpus(name: &str, source: &str, interval_ms: u64) -> CorpusResult { for (index, expression) in promql_lines(source).enumerate() { let normalized_expression = normalize(expression); let structural_shape = structural_shape(expression); - match lower_promql_with_sample_interval(expression, AccuracyTarget::exact(), interval_ms) { + match lower_promql_with_sample_interval(expression, interval_ms) { Ok(ir) => result.lowered.push(DumpRecord { corpus: name.to_string(), query_number: index + 1, @@ -392,14 +391,7 @@ async fn run_sql_corpora(out_dir: PathBuf) { for (index, expression) in queries.into_iter().enumerate() { let normalized_expression = normalize(&expression); let structural_shape = structural_shape(&expression); - match lower_sql_dialect( - &expression, - &catalog, - dialect.clone(), - AccuracyTarget::exact(), - ) - .await - { + match lower_sql_dialect(&expression, &catalog, dialect.clone()).await { Ok(ir) => lowered.push(DumpRecord { corpus: name.to_string(), query_number: index + 1, diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 4e231d027..44857bc46 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -53,23 +53,17 @@ // - stage0_query_language_frontends: the frontends' DAG, one root per query; // - stage1_logical_asap_optimization (from `stage1_pass1`): `pruned_candidates`, // the ways to compute a measure Pass 1 pruned (each on its aggregate's -// memo `group`, with `candidate` and `reason`); with `stage3`, -// `candidates` holds `L1`, the selected plan's roots (as Stage 2 built -// them, with Stage 3's weight proofs); `error` when Pass 1 failed; +// memo `group`, with `candidate` and `reason`); `error` when Pass 1 +// failed; // - stage2_physical_asap_optimization (from `stage2`): `pruned_candidates`, // the physical candidates Stage 2 pruned (each with its memo `group`, // `operator_node`, `operator`, `implementation_root`, `timing`, -// `materialization` and `reason`); with `stage3`, `candidates` holds -// `P1`, the selected plan as Stage 2 built it; -// - stage3_plan_selection (from `stage3`): Stage 3's `Selection`: -// `selected_dag` (`candidate_of_node`, `edges`, -// `node_of_output_reference`), `nodes_with_proven_non_negative_weights`, -// the plan's `cost` (its `usage`, `cost_per_second` and per-node -// charges), `accepted_accuracy_per_query`, `pruned_candidates` and -// `search_report`; `error` when no plan was selected. -// `tumbling_window_retention_ms` maps each tumbling-window chain the plan -// keeps beyond one evaluation (by node id in `P1`) to how far back its -// tumbling windows are kept. +// `lifetime` and `reason`); +// - stage3_plan_selection (from `stage3`): the `plan` Stage 3 built (a +// `PhysicalASAPDAG`) and its `Selection`: `accepted_accuracy_per_query`, +// `pruned_candidates`, `milp_variables`, `milp_constraints`, +// `milp_construction_ms` and `milp_solve_ms`; `error` when no plan was +// selected. // `materialized_views` are the plan's materialized views as // `plan_installable` derives them, each with the node of `P1` it is // installed from (`stage2_node_id`), its position among that node's @@ -97,11 +91,10 @@ // A section of a stage after `--run-until` is null. use std::collections::btree_map::Entry; -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::rc::Rc; +use std::collections::BTreeSet; -use asap_cost::{CostInputs, ResourceUsage}; -use asap_devtools::candidate_labels::{computation_label, node_label}; +use asap_cost::CostInputs; +use asap_devtools::candidate_labels::node_label; use asap_devtools::planning_stages::{ lower_promql_roots, lower_sql_roots, run_stages_until, sql_catalog, PlanningStagesError, PlanningStagesRun, RunUntil, @@ -109,18 +102,16 @@ use asap_devtools::planning_stages::{ use asap_devtools::planning_stages_text::planning_stages_text; use asap_devtools::SqlCatalog; use asap_frontend_promql::{metric_type_warnings, promql_metric_names}; -use asap_logical_asap_optimization::{ChosenCandidatePerMemoGroup, DeploymentCapabilities}; use asap_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection}; -use asap_planner::{ - installable_plan_of, selected_chosen_candidates, PlanningModels, PlanningRun, SelectionError, -}; -use asap_types::ir::flat::{flatten, FlatDag, FlatNode, NodeId}; -use asap_types::ir::operator::node::OperatorNode; -use asap_types::ir::properties::{default_metric_for_root, AccuracyTarget}; +use asap_planner::{installable_plan_of, PlanningModels, SelectionError}; +use asap_types::accuracy::{default_metric_for_root, AccuracyTarget}; +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::flat::{flatten, FlatDag, FlatNode}; use asap_types::ir::QueryRoot; -use asap_types::stages::{queries_rooted_at, MemoGroupId}; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; +use asap_types::stages::PhysicalASAPMemoCandidates; +use asap_types::stages::{queries_rooted_at, MemoGroup, MemoGroupId}; use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{PerComputationLogicalMemoCandidates, PhysicalASAPMemoCandidates}; use asap_types::workload::{ DataArrival, DataWorkload, PlanningWorkload, QueryLanguage, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, @@ -242,11 +233,11 @@ fn run(args: Vec) -> Result<(), Failure> { .as_ref() .ok() .and_then(|run| run.run.selection.as_ref().ok()) - .and_then(|selection| { - large_peak_memory_warning( - selection.cost.usage.peak_memory_bytes, - &cost_inputs.cost_model, - ) + .map(|(_plan, _selection)| -> u64 { + todo!("#1207 PR 6: the selected plan's peak memory, from plan_cost(&plan)") + }) + .and_then(|peak_memory_bytes| { + large_peak_memory_warning(peak_memory_bytes, &cost_inputs.cost_model) }), ); let text = if json_format { @@ -309,10 +300,7 @@ fn document_of( "workload": { "queries": queries }, "stage0_query_language_frontends": { "dag": export(roots) }, "stage1_logical_asap_optimization": { - "combinations": 0, - "capped": false, "pruned_candidates": [], - "candidates": [], "error": format!("Pass 1: {error}"), }, "stage2_physical_asap_optimization": null, @@ -326,31 +314,8 @@ fn document_of( /// section of a stage `run` ran, the others null. fn document(queries: Vec, entries: &[QueryWorkloadEntry], run: &PlanningStagesRun) -> Value { let until = run.run_until; - let labels = asap_devtools::candidate_labels::candidate_labels(run); - let logical: Vec<_> = run - .logical_candidates() - .iter() - .map(|c| match &c.roots { - Ok(roots) => { - json!({ "id": c.id, "display_name": labels[&c.id], "dag": export(roots) }) - } - Err(error) => { - json!({ "id": c.id, "display_name": labels[&c.id], "error": error }) - } - }) - .collect(); - let physical: Vec<_> = run - .physical_candidates() - .iter() - .map(|p| json!({ "id": p.id, "from_logical": p.from_logical, "display_name": labels[&p.id], "dag": p.dag })) - .collect(); let stage1 = (until >= RunUntil::Pass1).then(|| { - let mut stage1 = json!({ - "combinations": logical.len(), - "capped": false, - "pruned_candidates": pass1_pruned_candidates_json(run.run.per_computation_candidates.as_ref()), - "candidates": logical, - }); + let mut stage1 = json!({}); // Pass 1 or Pass 2 failed, or the deployment's capabilities // contradict the data workload. if run.run.logical_asap_candidates.is_none() { @@ -364,7 +329,6 @@ fn document(queries: Vec, entries: &[QueryWorkloadEntry], run: &PlanningS }); let stage2 = (until >= RunUntil::Stage2).then(|| { let mut stage2 = json!({ - "candidates": physical, "pruned": [], "pruned_candidates": stage2_pruned_candidates_json(run.run.physical_asap_candidates.as_ref()), }); @@ -389,47 +353,32 @@ fn document(queries: Vec, entries: &[QueryWorkloadEntry], run: &PlanningS }) } -/// Stage 3's section: the selection, the selected plan's tumbling-window retention and -/// its materialized views. +/// Stage 3's section: the plan, the selection and the plan's materialized +/// views. fn stage3_json(run: &PlanningStagesRun, entries: &[QueryWorkloadEntry]) -> Value { let mut stage3 = selection_json(&run.run.selection); - match (&run.run.selection, &run.run.selected_plan) { - (Ok(selection), Some(roots)) => { - let tumbling_window_retention_ms = run - .run - .selected_physical_plan - .as_ref() - .map(|realized| realized.tumbling_window_retention_ms.clone()) - .unwrap_or_default(); - stage3["tumbling_window_retention_ms"] = tumbling_window_retention_ms - .iter() - .map(|(node, ms)| (node.to_string(), json!(ms))) - .collect::>() - .into(); - match materialized_views_json(roots, selection, &tumbling_window_retention_ms, entries) - { - Ok(views) => stage3["materialized_views"] = Value::Array(views), - Err(error) => { - stage3["materialized_views"] = json!([]); - stage3["materialized_views_error"] = json!(error); - } + match &run.run.selection { + Ok((plan, _)) => match materialized_views_json(plan, entries) { + Ok(views) => stage3["materialized_views"] = Value::Array(views), + Err(error) => { + stage3["materialized_views"] = json!([]); + stage3["materialized_views_error"] = json!(error); } - } - _ => stage3["materialized_views"] = json!([]), + }, + Err(_) => stage3["materialized_views"] = json!([]), } stage3 } -/// The document's `memo` section: the memo after each step `run` ran, and -/// with Stage 3 the selected plan's choice over the last step's memo. +/// The document's `memo` section: the memo after each step `run` ran. fn memo_json(run: &PlanningStagesRun) -> Value { let stage0 = run .stage0_memo_groups .as_ref() .expect("run_stages_until builds Stage 0's memo"); let mut named = vec![("stage0", "Stage 0, the frontends' DAG hash-consed", stage0)]; - if let Some(pass1) = &run.run.per_computation_candidates { - named.push(("pass1", "Stage 1, Pass 1", &pass1.memo)); + if let Some(pass1) = &run.run.pass1_memo { + named.push(("pass1", "Stage 1, Pass 1", pass1)); } if let Some(pass2) = &run.pass2_local_window_composition { named.push(( @@ -452,33 +401,7 @@ fn memo_json(run: &PlanningStagesRun) -> Value { })); previous = Some(memo); } - let mut memo = json!({ "nodes": table.nodes, "steps": json_steps }); - if run.run_until == RunUntil::Stage3 { - match selected_choice(&run.run) { - Ok(choice) => memo["selected_choice"] = choice_json(&choice), - Err(reason) => { - memo["selected_choice"] = Value::Null; - memo["selected_choice_unavailable"] = json!(reason); - } - } - } - memo -} - -/// The selected plan's candidate for each group of the Pass 2 memo -/// (`run.logical_asap_candidates`), or why there is none. -fn selected_choice(run: &PlanningRun) -> Result { - match ( - &run.selection, - &run.logical_asap_candidates, - &run.physical_asap_candidates, - ) { - (Ok(selection), Some(logical), Some(physical)) => { - selected_chosen_candidates(&logical.memo, physical, &selection.selected_dag) - } - (Err(error), _, _) => Err(format!("no plan was selected: {error}")), - _ => Err("no plan was selected".into()), - } + json!({ "nodes": table.nodes, "steps": json_steps }) } /// The [`metric_type_warnings`] of each PromQL query of `described` (the @@ -615,21 +538,37 @@ fn lowered_together( } /// The arrival of two files' data planned together: ingesting (or mixed) -/// at the sum of their rates; the same arrival when both are at rest or -/// unknown; `None` when they differ. +/// at the sum of their rates, as late as the later of their out-of-order +/// bounds; the same arrival when both are at rest or unknown; `None` when +/// they differ. fn arrival_together(earlier: DataArrival, later: DataArrival) -> Option { match (earlier, later) { ( - DataArrival::Ingesting { rows_per_second: a }, - DataArrival::Ingesting { rows_per_second: b }, + DataArrival::Ingesting { + rows_per_second: a, + max_out_of_order_ms: late_a, + }, + DataArrival::Ingesting { + rows_per_second: b, + max_out_of_order_ms: late_b, + }, ) => Some(DataArrival::Ingesting { rows_per_second: a + b, + max_out_of_order_ms: late_a.max(late_b), + }), + ( + DataArrival::Mixed { + rows_per_second: a, + max_out_of_order_ms: late_a, + }, + DataArrival::Mixed { + rows_per_second: b, + max_out_of_order_ms: late_b, + }, + ) => Some(DataArrival::Mixed { + rows_per_second: a + b, + max_out_of_order_ms: late_a.max(late_b), }), - (DataArrival::Mixed { rows_per_second: a }, DataArrival::Mixed { rows_per_second: b }) => { - Some(DataArrival::Mixed { - rows_per_second: a + b, - }) - } (DataArrival::AtRest, DataArrival::AtRest) => Some(DataArrival::AtRest), (DataArrival::Unknown, DataArrival::Unknown) => Some(DataArrival::Unknown), _ => None, @@ -645,21 +584,10 @@ fn arrival_together(earlier: DataArrival, later: DataArrival) -> Option, + plan: &asap_types::plan::physical_asap_dag::PhysicalASAPDAG, entries: &[QueryWorkloadEntry], ) -> Result, String> { - let (plan, report) = installable_plan_of( - roots, - Some(selection), - tumbling_window_retention_ms, - entries, - ) - .map_err(|e| e.to_string())?; + let (plan, report) = installable_plan_of(plan, None, entries).map_err(|e| e.to_string())?; let views = plan.materialized_views().map_err(|e| e.to_string())?; let mut stage2_of = std::collections::HashMap::new(); for node in report @@ -689,106 +617,25 @@ fn materialized_views_json( .collect() } -/// The ways to compute a measure Pass 1 pruned -/// (`PerComputationLogicalMemoCandidates::pruned_candidates`), in pruning -/// order: each with its aggregate's memo `group`, the `candidate` and the -/// `reason` (`PruneReason`, e.g. `UnsupportedByDeployment`). Empty when -/// Pass 1 failed. -fn pass1_pruned_candidates_json(pass1: Option<&PerComputationLogicalMemoCandidates>) -> Vec { - pass1 - .map_or(&[][..], |pass1| &pass1.pruned_candidates) - .iter() - .map(|pruned| { - json!({ - "group": pruned.group.0, - "candidate": computation_label(&pruned.candidate), - "reason": format!("{:?}", pruned.reason), - }) - }) - .collect() -} - /// The physical candidates Stage 2 pruned -/// (`PhysicalASAPMemoCandidates::pruned_candidates`), in pruning order: -/// each with its memo `group`, its `operator_node` (index into Stage 2's -/// operator nodes), the node's `operator` label, whether it is an -/// `implementation_root`, its `timing` and `materialization`, and the -/// `reason` (`PruneReason`, e.g. -/// `RawDataNotRetained`). Empty without memo steps. +/// (`PhysicalASAPMemoCandidates::pruned_candidates`), in pruning order. +/// Empty without memo steps. fn stage2_pruned_candidates_json(stage2: Option<&PhysicalASAPMemoCandidates>) -> Vec { let Some(stage2) = stage2 else { return Vec::new(); }; - stage2 - .pruned_candidates - .iter() - .map(|pruned| { - let key = &pruned.candidate; - let node = &stage2.operator_nodes_of_candidates_for_memo_groups[key.operator_node.0]; - let flat = export(&[QueryRoot::Operator(node.physical_node.clone())]); - json!({ - "group": pruned.group.0, - "operator_node": key.operator_node.0, - "operator": flat.nodes.last().map(node_label).unwrap_or_default(), - "implementation_root": node.is_implementation_root, - "timing": key.timing, - "materialization": key.materialization, - "reason": format!("{:?}", pruned.reason), - }) - }) - .collect() + let _ = &stage2.pruned_candidates; + todo!("#1207 PR 9 (#1205): each pruned PhysicalCandidate with its reason") } -/// One plan's cost: per second, its predicted CPU, unit and source, its -/// usage and each node's charge. -fn cost_json(cost: &asap_cost::PlanCost) -> Value { - let per_node: serde_json::Map<_, _> = cost - .per_node - .iter() - .map(|(node, c)| { - ( - node.to_string(), - json!({ - "operator": c.operator, - "timing": c.timing, - "cpu_core_seconds_per_second": c.cpu_core_seconds_per_second, - "cost_per_second": c.cost_per_second, - "multiplier": c.multiplier, - "detail": c.detail, - "price_source": c.price_source, - }), - ) - }) - .collect(); - json!({ - "cost_per_second": cost.cost_per_second, - // Report-only: the nodes' predicted CPU, summed as - // `plan_installable`'s predicted usage sums it. - "cpu_core_seconds_per_second": cost - .per_node - .values() - .map(|node| node.cpu_core_seconds_per_second) - .sum::(), - "cost_unit": cost.cost_unit, - "source": cost.source, - "usage": usage_json(&cost.usage), - "per_node": per_node, - }) -} - -/// Per query, the accuracy Stage 3 accepted: the target's metric, the -/// accepted metric, whether it was accepted regardless of its guarantee, -/// why the guarantee check would have rejected it, the sketches read and a -/// one-line summary. +/// Per query, the accuracy Stage 3 accepted: the accepted metric, the +/// sketches read and a one-line summary. fn accepted_json(queries: &[AcceptedAccuracy]) -> Value { queries .iter() .map(|accepted| { json!({ - "target_metric": accepted.target_metric, "accepted_metric": accepted.accepted_metric, - "accepted_regardless_of_guarantee": accepted.accepted_regardless_of_guarantee, - "guarantee_check_rejection": accepted.guarantee_check_rejection, "sketches": accepted.sketches, "summary": accepted.to_string(), }) @@ -796,57 +643,26 @@ fn accepted_json(queries: &[AcceptedAccuracy]) -> Value { .collect() } -/// Stage 3's selection ([`Selection`]): the selected DAG over the selected -/// round's Stage 2 candidates (one node per run of a candidate), the nodes -/// whose weights it proved non-negative, the plan's cost, the accuracy accepted per query, the -/// candidates it pruned and its search report; `error` when the facade -/// selected no plan. -fn selection_json(selection: &Result) -> Value { - let selection = match selection { - Ok(selection) => selection, - Err(error) => return json!({ "selected_dag": null, "error": error.to_string() }), - }; - let dag = &selection.selected_dag; - let nodes = |nodes: &[asap_types::stages::SelectedNodeId]| -> Vec { - nodes.iter().map(|node| node.0).collect() +/// Stage 3's plan and selection ([`Selection`]): the plan it built, the +/// accuracy accepted per query, the candidates it pruned and the selection +/// MILP's size and time; `error` when the facade selected no plan. +fn selection_json(selection: &Result<(PhysicalASAPDAG, Selection), SelectionError>) -> Value { + let (plan, selection) = match selection { + Ok(selected) => selected, + Err(error) => return json!({ "plan": null, "error": error.to_string() }), }; json!({ - "selected_dag": { - "candidate_of_node": dag.candidate_of_node.iter().map(|id| id.0).collect::>(), - "edges": dag - .edges - .iter() - .map(|e| json!({ "producer": e.producer.0, "consumer": e.consumer.0, "input": e.input })) - .collect::>(), - "node_of_output_reference": nodes(&dag.node_of_output_reference), - }, - "nodes_with_proven_non_negative_weights": - nodes(&selection.nodes_with_proven_non_negative_weights), - "cost": cost_json(&selection.cost), + "plan": plan, "accepted_accuracy_per_query": accepted_json(&selection.accepted_accuracy_per_query), "pruned_candidates": selection .pruned_candidates .iter() - .map(|p| json!({ "group": p.group.0, "candidate": p.candidate.0, "reason": format!("{:?}", p.reason) })) - .collect::>(), - "search_report": selection.search_report, - }) -} - -/// A candidate's resource usage, in the units its field names give. -fn usage_json(usage: &ResourceUsage) -> Value { - json!({ - "cpu_ops_per_second": usage.cpu_ops_per_second, - "scanned_bytes_per_second": usage.scanned_bytes_per_second, - "average_memory_bytes": usage.average_memory_bytes, - "peak_memory_bytes": usage.peak_memory_bytes, - "disk_bytes": usage.disk_bytes, - "peak_cpu_cores": usage.peak_cpu_cores, - "latency_per_query": usage - .latency_per_query - .iter() - .map(|l| json!({ "compaction_ms": l.compaction_ms, "query_ms": l.query_ms })) + .map(|p| json!({ "candidate": format!("{:?}", p.candidate), "reason": format!("{:?}", p.reason) })) .collect::>(), + "milp_variables": selection.milp_variables, + "milp_constraints": selection.milp_constraints, + "milp_construction_ms": selection.milp_construction_ms, + "milp_solve_ms": selection.milp_solve_ms, }) } @@ -910,69 +726,24 @@ fn with_default_metrics(mut queries: Vec, roots: &[QueryRoot]) -> Vec Value { - Value::Object( - choice - .iter() - .map(|(group, &candidate)| (group.0.to_string(), json!(candidate))) - .collect(), - ) -} - /// The `memo` section's node table (`memo.nodes`): each distinct node once, -/// children before parents, its children as indices into the table. Nodes -/// are matched by their serialized JSON (children included), since the -/// steps rebuild equal nodes as different `Rc`s. +/// children before parents, its children as indices into the table. #[derive(Default)] struct MemoNodes { nodes: Vec, - index_of: HashMap, -} - -impl MemoNodes { - /// Adds `roots` and every node below them. Returns each root's index - /// and, for each node of that DAG, the original node and its index. - fn add( - &mut self, - roots: &[Rc], - ) -> (Vec, Vec<(Rc, NodeId)>) { - let roots: Vec<_> = roots.iter().cloned().map(QueryRoot::Operator).collect(); - let (dag, originals) = flatten(&roots); - let mut index_in_table: Vec = Vec::with_capacity(dag.nodes.len()); - for mut node in dag.nodes { - node.operator = node.operator.map_children(|&child| index_in_table[child]); - let key = serde_json::to_string(&node).expect("a flat node serializes"); - let index = *self.index_of.entry(key).or_insert_with(|| { - self.nodes.push(node); - self.nodes.len() - 1 - }); - index_in_table.push(index); - } - let roots = dag - .roots - .iter() - .map(|root| match root { - QueryRoot::Operator(id) => index_in_table[*id], - QueryRoot::Scalar(_) => unreachable!("memo DAGs have operator roots"), - }) - .collect(); - (roots, originals.into_iter().zip(index_in_table).collect()) - } } -/// `memo`'s groups, each with its candidates, their -/// nodes added to `table`; a candidate is `added` when `previous` (the memo -/// at the step before) has no candidate of that group with the same node and -/// children. +/// `memo`'s groups, each with its candidates, their nodes added to +/// `table`; a candidate is `added` when `previous` (the memo at the step +/// before) has no such candidate. fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) -> Vec { let ids = |groups: &[MemoGroupId]| groups.iter().map(|g| g.0).collect::>(); - memo.groups + memo.groups() .iter() - .zip(&memo.candidates) .enumerate() - .map(|(id, (group, candidates))| { - let node = table.add(std::slice::from_ref(&group.logical_node)).0[0]; + .map(|(id, group)| { + let candidates = &group.candidates; + let node = group_node(table, group); let mut by_origin = serde_json::Map::new(); for candidate in candidates { let count = by_origin @@ -980,54 +751,21 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) .or_insert(json!(0)); *count = json!(count.as_u64().unwrap_or(0) + 1); } - let earlier = previous - .and_then(|memo| memo.candidates.get(id)) - .map_or(&[][..], Vec::as_slice); - let listed: Vec<_> = candidates - .iter() - .enumerate() - .map(|(index, candidate)| { - let (roots, originals) = table.add(std::slice::from_ref(&candidate.root_node)); - let group_of_node: serde_json::Map<_, _> = originals - .iter() - .filter_map(|(original, node)| { - candidate - .input_groups - .iter() - .find(|child| { - Rc::ptr_eq( - &memo.groups[child.0 as usize].logical_node, - original, - ) - }) - .map(|child| (node.to_string(), json!(child.0))) - }) - .collect(); - json!({ - "index": index, - "implements": std::iter::once(id as u32) - .chain(candidate.fused_groups.iter().map(|g| g.0)) - .collect::>(), - "children": ids(&candidate.input_groups), - "origin": origin_name(&candidate.origin), - "accuracy_budget_parts": candidate.accuracy_budget_parts, - "added": !earlier.iter().any(|before| { - Rc::ptr_eq(&before.root_node, &candidate.root_node) - && before.input_groups == candidate.input_groups - }), - "label": candidate_node_label(&table.nodes, roots[0], &group_of_node), - "node": roots[0], - "group_of_node": group_of_node, - }) - }) - .collect(); + let listed: Vec = if candidates.is_empty() { + Vec::new() + } else { + let _ = previous; + todo!( + "#1207 PR 9 (#1205): each candidate's expression, properties and requirements" + ) + }; json!({ "id": id, "label": node_label(&table.nodes[node]), - "children": ids(&group.child_groups), + "children": group.expression.children().into_iter().map(|g| g.0).collect::>(), "parents": ids(group.parent_groups()), "queries": group.reached_by_queries(), - "roots_of": queries_rooted_at(&memo.query_roots, MemoGroupId(id as u32)), + "roots_of": queries_rooted_at(&memo.queries, MemoGroupId(id as u32)), "node": node, "candidate_count": candidates.len(), "candidate_count_by_origin": by_origin, @@ -1038,12 +776,16 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) .collect() } +/// The node of `group`'s own operator, added to `table`. +fn group_node(table: &mut MemoNodes, group: &MemoGroup) -> usize { + let _ = (table, group); + todo!("#1207 PR 9: a group's node from its expression") +} + /// `OriginOfCandidateForMemoGroup` in snake case. fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { match origin { OriginOfCandidateForMemoGroup::Pass1 => "pass1", - OriginOfCandidateForMemoGroup::SharedBuild => "shared_build", - OriginOfCandidateForMemoGroup::SummaryResult => "summary_result", OriginOfCandidateForMemoGroup::SummaryInput => "summary_input", OriginOfCandidateForMemoGroup::TumblingWindows => "tumbling_windows", OriginOfCandidateForMemoGroup::Rollup => "rollup", @@ -1052,89 +794,3 @@ fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { } .into() } - -/// A candidate's own nodes (those above its child groups' nodes), root -/// first, as `node_label`s joined by " ← ". -fn candidate_node_label( - nodes: &[FlatNode], - root: NodeId, - group_of_node: &serde_json::Map, -) -> String { - let mut labels = Vec::new(); - let mut pending = vec![root]; - let mut seen = HashSet::new(); - while let Some(id) = pending.pop() { - if group_of_node.contains_key(&id.to_string()) || !seen.insert(id) { - continue; - } - labels.push(node_label(&nodes[id])); - pending.extend(nodes[id].operator.children().into_iter().rev().copied()); - } - labels.join(" ← ") -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A node two DAGs share is stored once in the memo's node table, even - /// when each DAG built it as its own `Rc`, and each DAG rebuilt from the - /// table equals its own `FlatDag` export. - #[test] - fn memo_node_table_stores_a_shared_node_once() { - let root = |query: &str| { - let workload: PlanningWorkload = serde_json::from_value(json!({ - "query_workload": {"language": "promql", "query_batch": [{ - "query": query, "invocations": 1, - "requirements": {"accuracy": {"max_error": 0.0, "failure_probability": 0.0}}, - "predictability": {"predictable": {"known_at": null}}, - }]}, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 1.0}}, - "sources": {"m": {"series_count": 10, "sample_interval": 15000, "metric_type": "gauge"}}, - }, - })) - .unwrap(); - match lower_promql_roots(&workload).unwrap().remove(0) { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => unreachable!(), - } - }; - let (sum, max) = (root("sum(m)"), root("max(m)")); - let mut table = MemoNodes::default(); - let (sum_root, _) = table.add(std::slice::from_ref(&sum)); - let after_sum = table.nodes.len(); - let (max_root, _) = table.add(std::slice::from_ref(&max)); - // `max(m)` adds only its own aggregate: the scan is already there. - assert_eq!(table.nodes.len(), after_sum + 1); - // The DAG below `root`, renumbered in table order. - let rebuilt = |root: NodeId| { - let mut reached = BTreeSet::new(); - let mut pending = vec![root]; - while let Some(id) = pending.pop() { - if reached.insert(id) { - pending.extend(table.nodes[id].operator.children().into_iter().copied()); - } - } - let local: HashMap<_, _> = reached.iter().enumerate().map(|(i, &id)| (id, i)).collect(); - let nodes: Vec<_> = reached - .iter() - .map(|&id| { - let mut node = table.nodes[id].clone(); - node.operator = node.operator.map_children(|child| local[child]); - node - }) - .collect(); - FlatDag { - nodes, - roots: vec![QueryRoot::Operator(local[&root])], - } - }; - for (node, root) in [(sum, sum_root[0]), (max, max_root[0])] { - assert_eq!( - serde_json::to_value(rebuilt(root)).unwrap(), - serde_json::to_value(export(&[QueryRoot::Operator(node)])).unwrap() - ); - } - } -} diff --git a/crates/devtools/src/bin/planner_fuzz.rs b/crates/devtools/src/bin/planner_fuzz.rs index ce59b86ba..7f0fc16ec 100644 --- a/crates/devtools/src/bin/planner_fuzz.rs +++ b/crates/devtools/src/bin/planner_fuzz.rs @@ -15,8 +15,8 @@ // is filed. // // `--relations` takes relation names or the letters a–o (default: all); -// `--search` is `candidate-connections` (the default search) or -// `exhaustive`; `--ignore-accuracy-guarantees` accepts every output +// `--search` is `candidate-connections` (the default search); +// `--ignore-accuracy-guarantees` accepts every output // regardless of its accuracy guarantee (guarantees are enforced by // default, as `plan_installable` does). A relation is violated when it misses by more // than max(`--relative-tolerance` (default 0.01) × cost, @@ -33,7 +33,7 @@ use asap_devtools::planner_fuzz::report::{run_campaign, serve, Campaign, WorkerR use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, SearchKind}; const USAGE: &str = "usage: planner_fuzz --seed --cases --time-budget-s \ -[--relations ,...] [--search candidate-connections|exhaustive] \ +[--relations ,...] [--search candidate-connections] \ [--seed-workload ]... [--ignore-accuracy-guarantees] [--relative-tolerance ] \ [--absolute-tolerance ] [--max-shrink-attempts ] [--case-timeout-s ] --out "; @@ -94,7 +94,6 @@ fn run(args: Vec) -> Result<(), String> { "--search" => { settings.search = match value()?.as_str() { "candidate-connections" => SearchKind::CandidateConnections, - "exhaustive" => SearchKind::Exhaustive, other => return Err(format!("unknown search {other}")), } } diff --git a/crates/devtools/src/bin/sketch_coverage.rs b/crates/devtools/src/bin/sketch_coverage.rs index f511b3497..4af160d1e 100644 --- a/crates/devtools/src/bin/sketch_coverage.rs +++ b/crates/devtools/src/bin/sketch_coverage.rs @@ -1,31 +1,25 @@ -// cargo run -p asap-devtools --bin sketch_coverage -- --sample-interval-ms [--max-error ] +// cargo run -p asap-devtools --bin sketch_coverage -- --sample-interval-ms // // Lowers every query in every corpus we have (mirrors `variant_coverage`'s -// corpus list exactly, so the two reports are directly comparable) with an -// *approximate* `AccuracyTarget`, runs Pass 1 of logical ASAP-aware -// optimization (`generate_per_computation_logical_memo_candidates`, every -// summary supported, no accuracy pruning) over each query, and +// corpus list exactly, so the two reports are directly comparable), runs +// Pass 1 of logical ASAP-aware optimization (`pass1_memo`, every summary +// supported) over each query, and // reports which summary candidates Pass 1 offers for it, plus the fraction of // lowered queries with at least one. // // `--sample-interval-ms ` is the sample interval declared for every // metric a query reads, which its bare instant selectors look back. -// `--max-error ` (ε, default 0.01) sets the `AccuracyTarget` every query in -// every corpus lowers with. Without an approximate target, Pass 1 offers no -// summary candidate. -use asap_accuracy::{IgnoreAccuracyGuarantees, NoAccuracyEvidence}; use asap_devtools::corpora::{ bgp_catalog, dqc_catalog, netflow_catalog, parse_sample_interval, promql_lines, sql_statements, }; use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog}; use asap_frontend_sql::lower_sql_dialect; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::generate_per_computation_logical_memo_candidates; -use asap_types::ir::properties::AccuracyTarget; +use asap_logical_asap_optimization::memo_of_queries; +use asap_logical_asap_optimization::pass1_memo; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; -use asap_types::workload::{DataWorkload, SqlDialect}; +use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; +use asap_types::workload::SqlDialect; use std::collections::BTreeSet; use std::rc::Rc; @@ -47,27 +41,23 @@ fn pct(n: usize, total: usize) -> String { } } -/// Run Pass 1 over each of one corpus's already-lowered queries (each -/// lowered with `accuracy`) and collect the summary candidates it offers +/// Run Pass 1 over each of one corpus's already-lowered queries and collect the summary candidates it offers /// for any of the query's targets. fn analyze_corpus( name: &'static str, roots: Vec<(String, Rc)>, failed: usize, - accuracy: &AccuracyTarget, ) -> CorpusCoverage { let lowered = roots.len(); let mut rejected = 0; let mut summaries = Vec::new(); for (id, root) in roots { - let Ok(pass1) = generate_per_computation_logical_memo_candidates( - generate_dag_of_logical_memo_groups(&[QueryRoot::Operator(root)]), - std::slice::from_ref(accuracy), - &IgnoreAccuracyGuarantees, - &NoAccuracyEvidence, - &DataWorkload::default(), - None, - ) else { + let Ok(memo) = memo_of_queries(&[QueryRoot::Operator(root)], &[None], &Default::default()) + else { + rejected += 1; + continue; + }; + let Ok(pass1) = pass1_memo(memo, None) else { rejected += 1; continue; }; @@ -76,9 +66,9 @@ fn analyze_corpus( // excluded. A way to compute a measure that Pass 1 offers but cannot // compose into a candidate is not counted. let mut offered = BTreeSet::new(); - for candidate in pass1.memo.candidates.iter().flatten() { + for candidate in pass1.groups().iter().flat_map(|group| &group.candidates) { if candidate.fused_groups.is_empty() { - summary_families(&candidate.root_node, &mut offered); + summary_family(&candidate.expression.operator, &mut offered); } } summaries.push((id, offered)); @@ -92,20 +82,17 @@ fn analyze_corpus( } } -/// Adds the family of every summary `node` and its descendants build to -/// `families`. -fn summary_families(node: &OperatorNode, families: &mut BTreeSet) { - if let Some( - ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) = node.asap() +/// Adds the family of the summary `operator` builds, if any, to `families`. +fn summary_family(operator: &Operator, families: &mut BTreeSet) { + if let Operator::ASAP( + ASAPOp::SummaryAgg { summary_type, .. } + | ASAPOp::SummaryTumblingWindowChain { summary_type, .. }, + ) = operator { - if let FieldDataType::Sketch(kind, _) = family { - families.insert(format!("{:?}", kind.algorithm())); + if let FieldDataType::Sketch(kind) = summary_type { + families.insert(format!("{:?}", kind)); } } - for child in node.children() { - summary_families(child, families); - } } fn covered(r: &CorpusCoverage) -> usize { @@ -133,24 +120,9 @@ fn report(r: &CorpusCoverage) { println!(); } -fn parse_max_error() -> f64 { - let mut args = std::env::args().skip(1); - while let Some(arg) = args.next() { - if arg == "--max-error" { - let raw = args.next().expect("--max-error requires a value"); - return raw - .parse() - .unwrap_or_else(|_| panic!("--max-error must be a float, got {raw:?}")); - } - } - 0.01 -} - #[tokio::main] async fn main() { - let max_error = parse_max_error(); let interval_ms = parse_sample_interval(); - let accuracy = AccuracyTarget::max_error(max_error); let mut results = Vec::new(); // ── PromQL corpora — same set as `variant_coverage` ── @@ -182,12 +154,12 @@ async fn main() { let mut roots = Vec::new(); let mut failed = 0; for (i, q) in promql_lines(corpus).enumerate() { - match lower_promql_with_sample_interval(q, accuracy.clone(), interval_ms) { + match lower_promql_with_sample_interval(q, interval_ms) { Ok(qe) => roots.push((format!("q{i}"), qe)), Err(_) => failed += 1, } } - results.push(analyze_corpus(name, roots, failed, &accuracy)); + results.push(analyze_corpus(name, roots, failed)); } // ── SQL corpora ── @@ -217,12 +189,12 @@ async fn main() { let mut roots = Vec::new(); let mut failed = 0; for (i, q) in sql_statements(corpus).into_iter().enumerate() { - match lower_sql_dialect(&q, &catalog, dialect.clone(), accuracy.clone()).await { + match lower_sql_dialect(&q, &catalog, dialect.clone()).await { Ok(qe) => roots.push((format!("q{i}"), qe)), Err(_) => failed += 1, } } - results.push(analyze_corpus(name, roots, failed, &accuracy)); + results.push(analyze_corpus(name, roots, failed)); } for r in &results { @@ -234,7 +206,7 @@ async fn main() { let total_rejected: usize = results.iter().map(|r| r.rejected).sum(); let total_covered: usize = results.iter().map(covered).sum(); - println!("=== global (max_error = {max_error}) ==="); + println!("=== global ==="); println!( "total lowered: {total_lowered}, total failed: {total_failed}, rejected by Pass 1: {total_rejected}" ); diff --git a/crates/devtools/src/bin/variant_coverage.rs b/crates/devtools/src/bin/variant_coverage.rs index 5c0458e1c..0555af167 100644 --- a/crates/devtools/src/bin/variant_coverage.rs +++ b/crates/devtools/src/bin/variant_coverage.rs @@ -11,7 +11,6 @@ use asap_devtools::corpora::{ }; use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog}; use asap_frontend_sql::lower_sql_dialect; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{OperatorNode, ScalarExpr}; use asap_types::workload::SqlDialect; use std::collections::BTreeSet; @@ -201,7 +200,7 @@ async fn main() { let mut lowered = 0; let mut failed = 0; for q in promql_lines(corpus) { - match lower_promql_with_sample_interval(q, AccuracyTarget::exact(), interval_ms) { + match lower_promql_with_sample_interval(q, interval_ms) { Ok(qe) => { walk(&qe, &mut variants); lowered += 1; @@ -229,14 +228,7 @@ async fn main() { let mut lowered = 0; let mut failed = 0; for q in sql_statements(corpus) { - match lower_sql_dialect( - &q, - &catalog, - SqlDialect::DataFusionSQL, - AccuracyTarget::exact(), - ) - .await - { + match lower_sql_dialect(&q, &catalog, SqlDialect::DataFusionSQL).await { Ok(qe) => { walk(&qe, &mut variants); lowered += 1; @@ -262,14 +254,7 @@ async fn main() { let mut lowered = 0; let mut failed = 0; for q in sql_statements(corpus) { - match lower_sql_dialect( - &q, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - { + match lower_sql_dialect(&q, &catalog, SqlDialect::ClickhouseSQL).await { Ok(qe) => { walk(&qe, &mut variants); lowered += 1; diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index 87ef5d797..8372d36ae 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -2,101 +2,12 @@ //! `asap_planner` viewer document and text //! only. Planning never reads them: they //! are pure functions of the candidates' planning data (the logical roots -//! and the materialization assignment). - -use std::collections::BTreeMap; -use std::rc::Rc; +//! and the physical properties assignment). use serde_json::Value; -use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; -use asap_types::ir::NonASAPOp; -use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::physical::Materialization; -use asap_types::physical::PhysicalWorkloadPlan; use asap_types::stages::HowToComputeAggregateMeasure; - -use crate::planning_stages::PlanningStagesRun; - -/// The label of every logical and physical candidate the document shows -/// of `run` ([`PlanningStagesRun::logical_candidates`], -/// [`PlanningStagesRun::physical_candidates`]), by id. -pub fn candidate_labels(run: &PlanningStagesRun) -> BTreeMap { - let mut labels = BTreeMap::new(); - for candidate in run.logical_candidates() { - let label = candidate - .roots - .as_deref() - .map(connected_candidate_label) - .unwrap_or_else(|_| "Unimplemented candidate".into()); - labels.insert(candidate.id.clone(), label); - } - for candidate in &run.physical_candidates() { - let logical = labels - .get(&candidate.from_logical) - .cloned() - .unwrap_or_default(); - labels.insert(candidate.id.clone(), physical_label(&logical, candidate)); - } - labels -} - -/// Joint selections carry their actual connected DAG rather than a Cartesian -/// choice vector. Derive labels from that DAG so a sketch is never called exact. -fn connected_candidate_label(roots: &[QueryRoot]) -> String { - let mut labels = Vec::new(); - let mut producers: BTreeMap)> = BTreeMap::new(); - for (query, root) in roots.iter().enumerate() { - let mut states = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for operator in root.operator_refs() { - for node in OperatorNode::reachable(operator) { - let identity = Rc::as_ptr(&node) as usize; - if !seen.insert(identity) { - continue; - } - if matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) { - let name = build_label(&node.operator); - if !states.contains(&name) { - states.push(name.clone()); - } - producers - .entry(identity) - .or_insert_with(|| (name, Vec::new())) - .1 - .push(query); - } else if matches!(node.non_asap(), Some(NonASAPOp::Scan { .. })) { - producers - .entry(identity) - .or_insert_with(|| ("Scan".into(), Vec::new())) - .1 - .push(query); - } - } - } - labels.push(format!( - "Q{} {}", - query + 1, - if states.is_empty() { - "exact".into() - } else { - states.join(" + ") - } - )); - } - let mut shared: Vec<_> = producers - .values() - .filter(|(_, readers)| readers.len() > 1) - .map(|(name, readers)| format!("shared {name} ({})", query_names(readers))) - .collect(); - shared.sort(); - labels.extend(shared); - labels.join(" · ") -} +use asap_types::summary_type::SketchAlgorithm; /// A Pass 1 per-computation candidate, e.g. "exact", "Sum acc" or "Kll". pub fn computation_label(candidate: &HowToComputeAggregateMeasure) -> String { @@ -110,152 +21,6 @@ pub fn computation_label(candidate: &HowToComputeAggregateMeasure) -> String { } } -/// `what` with the queries that read it, e.g. "Kll for Q1, Q3"; `what` -/// alone when none is found (a node later rules rebuilt). -fn for_queries(what: String, queries: &[usize]) -> String { - match queries.is_empty() { - true => what, - false => format!("{what} for {}", query_names(queries)), - } -} - -/// Query names for query indices, e.g. "Q1, Q3". -fn query_names(queries: &[usize]) -> String { - queries - .iter() - .map(|q| format!("Q{}", q + 1)) - .collect::>() - .join(", ") -} - -/// `ms` in the largest whole unit: "1m", "10s", "250ms". -fn duration_name(ms: u64) -> String { - match ms { - ms if ms % 3_600_000 == 0 => format!("{}h", ms / 3_600_000), - ms if ms % 60_000 == 0 => format!("{}m", ms / 60_000), - ms if ms % 1_000 == 0 => format!("{}s", ms / 1_000), - ms => format!("{ms}ms"), - } -} - -/// `logical` followed by what the candidate materializes, e.g. -/// "… · ingestion time: Kll for Q1, 5 × 1m tumbling windows · query time, shared: -/// DDSketch"; -/// just `logical` when nothing is. -pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String { - let mut parts: Vec<(&str, Vec)> = Vec::new(); - for (id, materialization) in &candidate.materialization { - let phase = match materialization { - Materialization::NotMaterialized => continue, - Materialization::MatAtIngestion(_) => "ingestion time", - Materialization::MatAtQuery(_) => "query time, shared", - Materialization::MatAtQueryKeptAcrossEvaluations(_) => "query time, kept", - }; - let node = candidate - .dag - .nodes - .iter() - .find(|n| n.id == *id) - .expect("an assigned node is in the DAG"); - let label = match &node.payload { - // Name the chain: chains of one family and tumbling-window count differ by - // their readers and tumbling-window width. - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => format!( - "{}, {tumbling_window_count} × {} tumbling windows", - for_queries( - state_label(&node.payload), - &dag_readers(&candidate.dag, *id) - ), - duration_name(*tumbling_window_width_ms) - ), - payload => build_label(payload), - }; - match parts.iter_mut().find(|(p, _)| *p == phase) { - Some((_, labels)) => labels.push(label), - None => parts.push((phase, vec![label])), - } - } - std::iter::once(logical.to_string()) - .chain( - parts - .into_iter() - .map(|(phase, labels)| format!("{phase}: {}", labels.join(", "))), - ) - .collect::>() - .join(" · ") -} - -/// A summary build, e.g. "Kll ×5 tumbling windows" or "Sum acc". -fn build_label(payload: &Operator) -> String { - match payload { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => { - format!( - "{} ×{tumbling_window_count} tumbling windows", - state_label(payload) - ) - } - _ => state_label(payload), - } -} - -/// The state a summary build or tumbling-window chain keeps, e.g. "Kll" or "Sum acc"; -/// any other operator's kind. -fn state_label(payload: &Operator) -> String { - match payload { - Operator::ASAP( - ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => match family { - FieldDataType::Sketch(kind, _) => sketch_name(kind.algorithm()), - FieldDataType::ExactAggregate(kind, _) => format!("{kind:?} acc"), - _ => payload.kind_name().to_string(), - }, - _ => payload.kind_name().to_string(), - } -} - -/// The queries (in workload order) whose root reaches node `id` of `dag`. -fn dag_readers(dag: &PhysicalASAPDAG, id: PhysicalASAPNodeId) -> Vec { - // Query `q` is the scalar root keyed `q` when there is one, and - // otherwise the next operator root. - let mut operator_roots = dag.roots.iter(); - let query_starts: Vec> = (0..dag.roots.len() + dag.scalar_roots.len()) - .map(|q| match dag.scalar_roots.iter().find(|(i, _)| *i == q) { - Some((_, expr)) => expr.operator_refs().into_iter().copied().collect(), - None => operator_roots.next().into_iter().copied().collect(), - }) - .collect(); - query_starts - .into_iter() - .enumerate() - .filter(|(_, starts)| { - let mut seen = std::collections::HashSet::new(); - let mut stack = starts.clone(); - while let Some(node) = stack.pop() { - if node == id { - return true; - } - if seen.insert(node) { - stack.extend( - dag.edges - .iter() - .filter(|e| e.consumer == node) - .map(|e| e.producer), - ); - } - } - false - }) - .map(|(q, _)| q) - .collect() -} - /// A sketch algorithm's display name, e.g. "CMS+heap" or "Kll". fn sketch_name(algorithm: &SketchAlgorithm) -> String { match algorithm { @@ -358,9 +123,7 @@ fn find_details(value: &Value, details: &mut Vec, depth: usize) { // A time range's length, e.g. `60s`. "range" => details.push(duration_text(field)), // A window, and a tumbling-window chain's layout. - "window" | "tumbling_window_width_ms" | "tumbling_window_count" => { - details.push(format!("{key}={field}")) - } + "window" | "tumbling_window_width_ms" => details.push(format!("{key}={field}")), _ => find_details(field, details, depth + 1), } } @@ -386,168 +149,3 @@ fn duration_text(value: &Value) -> String { _ => value.to_string(), } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::planner_layering_examples::example1; - use crate::planning_stages::lower_promql_roots; - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - - /// Selected DAG labels expose accumulator state without a legacy choice ID. - #[test] - fn connected_labels_describe_selected_states() { - let workload = example1(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &lower_promql_roots(&workload).unwrap(), - ), - ) - .unwrap(); - let choice = vec![0; inventory.targets.len()]; - let roots: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let label = connected_candidate_label(&roots); - assert!(label.contains("Q1 ") && label.contains("Q2 ")); - assert!(label.contains("acc"), "{label}"); - } - - /// A state reused by two query outputs is named once in the sharing suffix. - #[test] - fn connected_labels_name_actual_shared_state() { - let workload = example1(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &lower_promql_roots(&workload).unwrap(), - ), - ) - .unwrap(); - let roots = - compose_logical_candidate(&inventory, &vec![0; inventory.targets.len()]).unwrap(); - let label = connected_candidate_label(&[roots[1].clone(), roots[1].clone()]); - assert!(label.contains("shared Sum acc (Q1, Q2)"), "{label}"); - assert_eq!(label.matches("shared Sum acc").count(), 1); - } - - /// Two physical candidates that each maintain a different query's tumbling window - /// chain at ingestion time get different labels, even when both chains - /// have the same family and tumbling-window count (6 × 10s for Q1, 6 × 30s for Q2). - #[test] - fn physical_labels_name_the_tumbling_window_chain_maintained_at_ingestion() { - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_physical_asap_optimization::optimize_physical_asap; - use asap_types::workload::PlanningWorkload; - - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "query_batch": null, "repeating_queries": [ - {"query": "quantile_over_time(0.5, data[1m])", "demand": {"fixed_interval": 10000}, - "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, - "predictability": {"predictable": {"known_at": null}}}, - {"query": "quantile_over_time(0.5, data[3m])", "demand": {"fixed_interval": 30000}, - "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "gauge", "sample_interval": 15000}}}}"#, - ) - .unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &lower_promql_roots(&workload).unwrap(), - ), - ) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - *c == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch) - }) - .expect("a DDSketch candidate") - }) - .collect(); - let summary: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let grids: Vec<_> = entries.iter().map(|e| e.evaluation_grid_ms()).collect(); - let tumbling_windows = tumbling_window_candidates(&summary, &grids); - let roots = window_composition_pass2_candidates(&summary, &tumbling_windows) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty()) - .expect("each query's DDSketch in its own tumbling-window chain") - .roots - .unwrap(); - let plans = optimize_physical_asap( - "L1", - &roots, - &entries, - &workload.data_workload.clone().unwrap_or_default(), - false, - ) - .unwrap(); - // The plans that maintain exactly one tumbling-window chain at ingestion time. - let one_chain_at_ingestion: Vec = plans - .iter() - .filter(|plan| { - plan.materialization - .iter() - .filter(|(id, m)| { - matches!(m, Materialization::MatAtIngestion(_)) - && matches!( - plan.dag - .nodes - .iter() - .find(|n| n.id == **id) - .unwrap() - .payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .count() - == 1 - }) - .map(|plan| physical_label("L1", plan)) - .collect(); - let ingestion = |label: &String| { - label - .split(" · ") - .find(|p| p.starts_with("ingestion time")) - .map(str::to_string) - }; - assert!( - one_chain_at_ingestion - .iter() - .any(|l| ingestion(l).as_deref() - == Some("ingestion time: DDSketch for Q1, 6 × 10s tumbling windows")), - "{one_chain_at_ingestion:#?}" - ); - assert!( - one_chain_at_ingestion - .iter() - .any(|l| ingestion(l).as_deref() - == Some("ingestion time: DDSketch for Q2, 6 × 30s tumbling windows")), - "{one_chain_at_ingestion:#?}" - ); - let mut distinct = one_chain_at_ingestion.clone(); - distinct.sort(); - distinct.dedup(); - assert_eq!( - distinct.len(), - one_chain_at_ingestion.len(), - "{one_chain_at_ingestion:#?}" - ); - } - - /// Raw logical operators have an exact label, rather than an empty choice label. - #[test] - fn connected_labels_name_exact_results() { - let roots = lower_promql_roots(&example1()).unwrap(); - assert_eq!(connected_candidate_label(&roots), "Q1 exact · Q2 exact"); - } -} diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index f3c546d6e..2b1e21f71 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -23,7 +23,6 @@ pub mod candidate_labels; pub mod corpora; -pub mod memo_pricing; pub mod planner_fuzz; pub mod planner_layering_examples; pub mod planning_stages; @@ -40,7 +39,6 @@ pub use asap_frontend_sql::{lower_sql, SqlCatalog, SqlError}; /// default. pub fn lower_promql_with_sample_interval( query: &str, - accuracy: asap_types::ir::properties::AccuracyTarget, interval_ms: u64, ) -> Result, PromqlError> { use asap_types::workload::{ @@ -53,10 +51,7 @@ pub fn lower_promql_with_sample_interval( language: asap_types::workload::QueryLanguage::PromQL, query_batch: Some(vec![BatchEntry { query: Query(query.into()), - requirements: QueryRequirements { - accuracy, - ..Default::default() - }, + requirements: QueryRequirements::default(), predictability: Predictability::Unknown, invocations: 1, execute_at: None, diff --git a/crates/devtools/src/memo_pricing.rs b/crates/devtools/src/memo_pricing.rs deleted file mode 100644 index 128cc4911..000000000 --- a/crates/devtools/src/memo_pricing.rs +++ /dev/null @@ -1,401 +0,0 @@ -//! Price a memo's enumerated choices as the selected plan is priced: each -//! choice's composed DAG (`memo::enumeration::build_logical_asap_dags_from_chosen_candidates`) goes through Stage 2 -//! (`optimize_physical_asap`, every materialization assignment) and Stage 3 -//! (the reference pricer, `asap_plan_selection::reference_pricer`, under the -//! run's models), and the plan it selects for -//! it is installed as `plan_installable` installs one (`installable_plan_of`, -//! which also compiles it with the executor), so a choice cheaper than the -//! selected plan shows that the search missed a plan the memo stands for. - -use std::collections::HashMap; -use std::rc::Rc; - -use asap_logical_asap_optimization::{ - build_logical_asap_dags_from_chosen_candidates, ChosenCandidatePerMemoGroup, -}; -use asap_physical_asap_optimization::optimize_physical_asap; -use asap_plan_selection::reference_pricer::{select_among_plans, WholePlanSelection}; -use asap_plan_selection::{ErrorSelectingPlan, Selection}; -use asap_planner::{installable_plan_of, InstallablePlanError, PlanningModels, PlanningRun}; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot}; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::stages::Memo; -use asap_types::stages::MemoGroupId; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -/// One choice's result: valid with its cost, or why it is not valid. -#[derive(Debug, Clone, PartialEq)] -pub struct PricedChoice { - /// Stage 2 implemented it, Stage 3 accepted one of its physical plans, - /// and the plan Stage 3 selected installs (a standalone scalar query, - /// which no choice changes, aside). - pub valid: bool, - /// Why it is not valid: the Stage 3 rejection cause of its first - /// physical plan in snake_case (e.g. `accuracy_target_missed`), or - /// `compose_failed`, `invalid_structure`, `stage2_failed`, - /// `stage3_failed`, `rejected` (no plan and no rejection listed), - /// `not_executable` (the executor cannot compile the selected plan's - /// view refresh or query run), or `not_installable` (installation - /// refuses the selected plan for another reason, e.g. an unsupported - /// raw-data window). - pub rejection: Option, - /// The rejection's one-line reason, with the numbers. - pub rejection_reason: Option, - /// The cheapest cost per second of its valid physical plans under the - /// run's cost model; `None` when not valid or under cost model 3. - pub cost_per_second: Option, - /// Valid and costs less than the selected plan (beyond rounding). - pub cheaper_than_selected: bool, -} - -/// What the choices are compared against: the run's selection. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct SelectedBaseline { - /// The selected plan's cost per second. - pub cost_per_second: Option, - /// The calibration factor the run priced with, frozen for every choice - /// so each is priced on the same scale as the selected plan. - pub calibration_factor: Option, -} - -impl SelectedBaseline { - /// The baseline of `selection`: its selected plan's cost. Stage 3 - /// reports no calibration factor, so a measured cost table without a - /// frozen factor is priced at each choice's own fit. - pub fn of(selection: &Selection) -> Self { - Self { - cost_per_second: selection.cost.cost_per_second, - calibration_factor: None, - } - } -} - -/// Costs within this fraction of each other are the same sum of charges -/// rounded differently (Stage 3's relative tie tolerance). -const TIE_TOLERANCE: f64 = 1e-9; - -/// Each of `choices` of `memo`, composed, implemented and priced. Query `i` -/// is `stage0_roots[i]` with workload entry `entries[i]`; its operator -/// references are replaced by their root groups' composed DAGs. -pub fn price_choices( - memo: &Memo, - choices: &[ChosenCandidatePerMemoGroup], - stage0_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, - selected: Option, -) -> Vec { - let mut settings = models.cost_inputs.clone(); - if settings.frozen_calibration_factor.is_none() { - settings.frozen_calibration_factor = selected.and_then(|s| s.calibration_factor); - } - let models = models.with_cost_inputs(&settings); - let selected_cost = selected.and_then(|s| s.cost_per_second); - // Which root group each query's operator references name is the same - // for every choice. - let root_groups = root_groups_of_queries(memo, stage0_roots); - choices - .iter() - .enumerate() - .map(|(index, choice)| { - let roots = build_logical_asap_dags_from_chosen_candidates(memo, choice) - .map_err(|error| error.to_string()) - .and_then(|composed| query_roots(&composed, stage0_roots, &root_groups)); - let mut priced = price_choice(roots, index, entries, data, models); - priced.cheaper_than_selected = match (priced.cost_per_second, selected_cost) { - (Some(cost), Some(selected)) => cost < selected - TIE_TOLERANCE * selected.abs(), - _ => false, - }; - priced - }) - .collect() -} - -/// One choice, its query roots composed (or why they do not compose), -/// implemented and priced. -fn price_choice( - roots: Result, String>, - index: usize, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> PricedChoice { - let invalid = |cause: &str, reason: String| PricedChoice { - valid: false, - rejection: Some(cause.into()), - rejection_reason: Some(reason), - cost_per_second: None, - cheaper_than_selected: false, - }; - let roots = match roots { - Ok(roots) => roots, - Err(error) => return invalid("compose_failed", error), - }; - if let Err(error) = roots.iter().try_for_each(QueryRoot::validate_structure) { - return invalid("invalid_structure", error.to_string()); - } - let physical = match physical_plans_of(&roots, index, entries, data, models) { - Ok(physical) => physical, - Err(error) => return invalid("stage2_failed", error), - }; - match reference_select(&physical, entries, data, models) { - Ok(selection) => { - match installable_plan_of( - &selection.plan, - None, - &selection.tumbling_window_retention_ms, - entries, - ) { - // A standalone scalar query, which a runtime cannot answer yet, - // does not depend on the choice. - Ok(_) | Err(InstallablePlanError::UnsupportedQueryRoot { .. }) => PricedChoice { - valid: true, - rejection: None, - rejection_reason: None, - cost_per_second: selection.costs[&selection.selected].cost_per_second, - cheaper_than_selected: false, - }, - Err( - error @ (InstallablePlanError::MaterializedViewNotExecutable { .. } - | InstallablePlanError::QueryRunNotExecutable { .. }), - ) => invalid("not_executable", error.to_string()), - Err(error) => invalid("not_installable", error.to_string()), - } - } - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => match rejected.first() { - Some(first) => invalid( - serde_json::to_value(first.cause) - .ok() - .and_then(|v| v.as_str().map(str::to_owned)) - .as_deref() - .unwrap_or("rejected"), - first.reason.clone(), - ), - None => invalid("rejected", "no valid physical plan".into()), - }, - Err(error) => invalid("stage3_failed", error.to_string()), - } -} - -/// Every physical plan Stage 2 offers for choice `index`'s composed `roots` -/// (each materialization assignment), named `E-`. -pub fn physical_plans_of( - roots: &[QueryRoot], - index: usize, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> Result, String> { - let id = format!("E{}", index + 1); - let mut physical = optimize_physical_asap( - &id, - roots, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - ) - .map_err(|error| error.to_string())?; - retain_deployable(&mut physical, models); - if physical.is_empty() { - return Err("no physical plan".into()); - } - for (k, plan) in physical.iter_mut().enumerate() { - plan.id = format!("{id}-{k}"); - plan.from_logical = id.clone(); - } - Ok(physical) -} - -/// Drops the plans of `physical` that Stage 2 would not offer under -/// `models.deployment_capabilities`: without raw data retained, a plan that -/// scans raw data at query time (Stage 2's `PruneReason::RawDataNotRetained`; -/// `optimize_physical_asap` enumerates every assignment and leaves this -/// rule to Stage 2's catalog, and the reference pricer no longer checks it). -pub fn retain_deployable(physical: &mut Vec, models: PlanningModels<'_>) { - if models.deployment_capabilities.raw_data_retained { - return; - } - physical.retain(|plan| { - plan.dag.nodes.iter().all(|node| { - !matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { .. }) - ) || node.output_state.timing == ExecutionTiming::IngestionTime - }) - }); -} - -/// The reference pricer over `physical` under `models`: every plan checked, -/// priced and ranked, every candidate kept. -pub fn reference_select( - physical: &[PhysicalWorkloadPlan], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> Result { - select_among_plans( - physical, - entries, - data, - &models.accuracy_check_inputs(), - models.cost_inputs, - ) -} - -/// One enumerated memo choice of a reference enumeration: its composed -/// logical roots (or why they do not compose), under id `E`. -#[derive(Debug)] -pub struct EnumeratedChoice { - pub id: String, - pub roots: Result, String>, -} - -/// The reference enumeration: the first `max_choices` choices of the -/// Pass 2 memo (`run.logical_asap_candidates`), each composed, -/// implemented by Stage 2 with every materialization assignment, and all -/// their physical plans priced together by the reference pricer. -#[derive(Debug)] -pub struct ReferenceEnumeration { - pub choices: Vec, - pub physical: Vec, - /// Why a composed choice has no physical plan, by choice id. - pub stage2_failed: Vec<(String, String)>, - /// Whether the memo has more choices than `max_choices`. - pub capped: bool, - pub selection: Result, -} - -/// The reference enumeration of `run` ([`ReferenceEnumeration`]); `None` -/// without a memo. -pub fn reference_enumeration( - run: &PlanningRun, - stage0_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, - max_choices: usize, -) -> Option { - use asap_logical_asap_optimization::enumerate; - let memo = &run.logical_asap_candidates.as_ref()?.memo; - let enumeration = enumerate(memo, max_choices.saturating_add(1)); - let capped = !enumeration.complete || enumeration.all_chosen_candidates.len() > max_choices; - let root_groups = root_groups_of_queries(memo, stage0_roots); - let mut choices = Vec::new(); - let mut physical = Vec::new(); - let mut stage2_failed = Vec::new(); - for (index, choice) in enumeration - .all_chosen_candidates - .iter() - .take(max_choices) - .enumerate() - { - let id = format!("E{}", index + 1); - let roots = build_logical_asap_dags_from_chosen_candidates(memo, choice) - .map_err(|error| error.to_string()) - .and_then(|composed| query_roots(&composed, stage0_roots, &root_groups)) - .and_then(|roots| { - roots - .iter() - .try_for_each(QueryRoot::validate_structure) - .map_err(|error| error.to_string())?; - Ok(roots) - }); - if let Ok(roots) = &roots { - match physical_plans_of(roots, index, entries, data, models) { - Ok(plans) => physical.extend(plans), - Err(error) => stage2_failed.push((id.clone(), error)), - } - } - choices.push(EnumeratedChoice { id, roots }); - } - let selection = reference_select(&physical, entries, data, models); - Some(ReferenceEnumeration { - choices, - physical, - stage2_failed, - capped, - selection, - }) -} - -/// For each query, the root group of each of its operator references, in -/// `operator_refs` order: the one group the query's root (in the memo's -/// `query_roots`) reads when its root is one operator, else the group -/// whose node is the reference. -fn root_groups_of_queries( - memo: &Memo, - stage0_roots: &[QueryRoot], -) -> Result>, String> { - stage0_roots - .iter() - .enumerate() - .map(|(query, root)| { - let groups: Vec<_> = memo - .query_roots - .get(query) - .map(|root| { - root.operator_refs() - .into_iter() - .copied() - .collect::>() - }) - .unwrap_or_default() - .into_iter() - .collect(); - match root { - QueryRoot::Operator(_) if groups.len() == 1 => Ok(groups), - QueryRoot::Operator(node) => Ok(vec![group_of(memo, &groups, node, query)?]), - QueryRoot::Scalar(expression) => expression - .operator_refs() - .into_iter() - .map(|node| group_of(memo, &groups, node, query)) - .collect(), - } - }) - .collect() -} - -/// The group among `groups` whose node is `node`: the same node, else an -/// equal one (the memo's group nodes are the frontends' nodes with common -/// sub-DAGs shared). -fn group_of( - memo: &Memo, - groups: &[MemoGroupId], - node: &Rc, - query: usize, -) -> Result { - let node_of = |g: &&MemoGroupId| &memo.groups[g.0 as usize].logical_node; - groups - .iter() - .find(|g| Rc::ptr_eq(node_of(g), node)) - .or_else(|| groups.iter().find(|g| **node_of(g) == **node)) - .copied() - .ok_or_else(|| format!("query {query}'s root has no root group")) -} - -/// Each query's root with every operator reference replaced by the composed -/// DAG of its root group (`root_groups`). -fn query_roots( - composed: &[(MemoGroupId, Rc)], - stage0_roots: &[QueryRoot], - root_groups: &Result>, String>, -) -> Result, String> { - let root_groups = root_groups.as_ref().map_err(String::clone)?; - let composed: HashMap<_, _> = composed.iter().cloned().collect(); - stage0_roots - .iter() - .zip(root_groups) - .map(|(root, groups)| { - let mut next = groups.iter().map(|group| Rc::clone(&composed[group])); - Ok(match root { - QueryRoot::Operator(_) => QueryRoot::Operator(next.next().expect("one group")), - QueryRoot::Scalar(expression) => QueryRoot::Scalar( - expression.map_operator_refs(&mut |_| next.next().expect("a group per ref")), - ), - }) - }) - .collect() -} diff --git a/crates/devtools/src/planner_fuzz/README.md b/crates/devtools/src/planner_fuzz/README.md index 3370fa96f..56ebb9825 100644 --- a/crates/devtools/src/planner_fuzz/README.md +++ b/crates/devtools/src/planner_fuzz/README.md @@ -106,7 +106,7 @@ derive_cost = cost(witness) − cost(W's plan) never from a guess: 1. Stage 2 and Stage 3 implement and price the witness the same way they price - any candidate. Each materialization assignment goes through + any candidate. Each physical properties assignment goes through `optimize_physical_asap` and `select_plan`, using the same cost model, deployment capabilities and accuracy checks as the search. The cheapest assignment is kept. 2. `W`'s selected logical plan is priced the same way. This gives the witness @@ -137,7 +137,7 @@ Two rules decide whether a check counts: | `e-other-quantile-rank` | `q` at another rank | `q`'s estimate (or exact quantile aggregate) at the other rank, same input | | `f-exact-duplicate` | `q` | reading `q`'s node | | `g-outer-over-result` | an operator over `q`, by `q`'s outer operator: `topk(3, q)` (sum), `bottomk(3, q)` (count), `max(q)` (max), `quantile(0.5, q)` (avg), `q > 0` (min, quantile) | the operator's lowering grafted onto `q`'s node | -| `h-other-statistic` | SQL, ungrouped: another frequency statistic of `q`'s column (distinct count → row count, L2 → distinct count, entropy → L2) | a `SummaryEstimate` of it from `q`'s state, when its family answers it (`FamilyCapability::answers`); or `W ∪ {q'}` planned with UnivMon as the only sketch family, priced with every family | +| `h-other-statistic` | SQL, ungrouped: another frequency statistic of `q`'s column (distinct count → row count, L2 → distinct count, entropy → L2) | a `SummaryEstimate` of it from `q`'s state, when its family answers it (`SummaryTypeCapability::answers`); or `W ∪ {q'}` planned with UnivMon as the only sketch family, priced with every family | | `i-other-repeat-interval` | `q` at a longer repeat interval | reading `q`'s node | | `j-smaller-topk` | `q`'s top-k at k = 3 | `q`'s `Limit` / `TopK` at 3 over the same input | | `k-union-of-selections` | `q` over `label =~ "v1\|v2"`, with `q`'s `v2` twin added to `W` | `SummaryMerge` of the two states, then `q`'s estimate | @@ -204,7 +204,7 @@ status column, "conclusive" means the check either held or was violated. | Another quantile rank from the same sketch | planner-layering.md L631–634, L804–817 | `e-other-quantile-rank` | VIOLATED (39 of 465), exact percentiles only: new draft (unshared exact aggregates) | | Identical expressions (common-subexpression elimination) | planner-layering.md rule table L583–585, Algorithm 2 L655–658 | `f-exact-duplicate` | VIOLATED (92 of 2,915), exact query-time aggregates: new draft | | A query over another query's result (top-k over a shared sum) | joint-planning-search.md L177–187; planner-layering.md L583–585 | `g-outer-over-result` | holds (552 conclusive checks). With the operator following q's (ASAPWorkloads #4, #12, #17, #18): VIOLATED (6 of 49), `topk(3, …)` over an exact instant sum and `… > 0` over an approximate quantile | -| Summary capability: another statistic the family answers (`FamilyCapability::answers`) | planner-layering.md rule table L586, L611–617, Example 2 L1306–1370 | `h-other-statistic` | KLL and KMV answer one statistic, so h now checks SQL frequency statistics of a high-cardinality column (planner-layering Example 2; ASAPWorkloads #57, #107–#108): holds (5 conclusive checks) | +| Summary capability: another statistic the family answers (`SummaryTypeCapability::answers`) | planner-layering.md rule table L586, L611–617, Example 2 L1306–1370 | `h-other-statistic` | KLL and KMV answer one statistic, so h now checks SQL frequency statistics of a high-cardinality column (planner-layering Example 2; ASAPWorkloads #57, #107–#108): holds (5 conclusive checks) | | gcd tumbling-window grid across repeat intervals | planner-layering.md L791–809 | `i-other-repeat-interval` | VIOLATED (49 of 2,717): new draft; compare #731 | | Exact top-k reuse (smaller k) | planner-layering.md rule table L589; joint-planning-search.md L325 | `j-smaller-topk` | holds (62 conclusive checks) | | Merge of disjoint selections | asap-primitive-schema.md §4.2 L194–232, §5.6 Example B L721–752 | `k-union-of-selections` | never conclusive: Stage 1 composes no guarantee for the merged estimate | diff --git a/crates/devtools/src/planner_fuzz/generate.rs b/crates/devtools/src/planner_fuzz/generate.rs index d831c9894..9efc183c3 100644 --- a/crates/devtools/src/planner_fuzz/generate.rs +++ b/crates/devtools/src/planner_fuzz/generate.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use serde_json::json; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use super::{FuzzQuery, FuzzWorkload, Language}; @@ -376,7 +376,7 @@ pub fn source_facts( let mut facts = json!({ "series_count": series_count, "sample_interval": sample_interval_ms, - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "distinct_values_per_label": distinct, }); if let Some(metric_type) = metric_type { diff --git a/crates/devtools/src/planner_fuzz/mod.rs b/crates/devtools/src/planner_fuzz/mod.rs index e37b85e75..cea78b165 100644 --- a/crates/devtools/src/planner_fuzz/mod.rs +++ b/crates/devtools/src/planner_fuzz/mod.rs @@ -8,8 +8,7 @@ //! target, a filter on a group key, a coarser grouping, ...), it builds a //! *witness*: `W`'s selected logical plan plus `q'` derived from `q`'s //! node by one operator. Stage 2 and Stage 3 implement and price the -//! witness exactly as they price any candidate -//! ([`optimize_physical_asap`], the reference pricer), and the relation +//! witness exactly as they price any candidate, and the relation //! //! ```text //! cost(W ∪ {q'}) − cost(W) ≤ derive_cost + tolerance, @@ -35,27 +34,24 @@ pub mod relations; pub mod report; pub mod shrink; -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::HashMap; use std::rc::Rc; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use asap_cost::CostInputs; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_physical_asap_optimization::optimize_physical_asap; use asap_planner::{PlanSearchSettings, PlanningModels}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::sketch::SketchAlgorithm; +use asap_types::accuracy::AccuracyTarget; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::QueryRoot; +use asap_types::summary_type::SketchAlgorithm; use asap_types::workload::{ DataArrival, DataWorkload, PlanningWorkload, QueryLanguage, QueryWorkload, RepeatingEntry, SqlDialect, }; -use crate::planning_stages::{ - lower_promql_roots, lower_sql_roots, run_planning_stages_over, run_search_over, sql_catalog, -}; +use crate::planning_stages::{lower_promql_roots, lower_sql_roots, run_search_over, sql_catalog}; use generate::QuerySpec; /// The language of a fuzz workload: one per workload, as one @@ -172,7 +168,10 @@ impl FuzzWorkload { } } if rows_per_second > 0.0 { - self.data.arrival = DataArrival::Ingesting { rows_per_second }; + self.data.arrival = DataArrival::Ingesting { + rows_per_second, + max_out_of_order_ms: None, + }; } } @@ -208,7 +207,7 @@ impl FuzzWorkload { .filter(|supported| { self.sketch_families .as_ref() - .is_none_or(|only| only.contains(&supported.family)) + .is_none_or(|only| only.contains(&supported.summary_type)) }) .collect(), ), @@ -219,17 +218,9 @@ impl FuzzWorkload { /// The `asap_planner` invocation (a bash command) that plans this /// workload, saved as `workload_file`, with `settings`' accuracy - /// guarantees. `asap_planner` has no exhaustive search, so under - /// `--search exhaustive` the command starts with a comment saying it - /// may not reproduce. Its output document is `.out.json`. + /// guarantees. Its output document is `.out.json`. pub fn asap_planner_command(&self, workload_file: &str, settings: &FuzzSettings) -> String { let mut command = String::new(); - if matches!(settings.search, SearchKind::Exhaustive) { - command.push_str( - "# asap_planner runs the candidate-connection search, not the \ - exhaustive one; it may not reproduce this.\n", - ); - } command.push_str(&format!( "cargo run -p asap-devtools --bin asap_planner -- --workload {workload_file} \ --run-until stage3 --format json" @@ -314,14 +305,8 @@ pub fn read_seed_file(path: &std::path::Path) -> Result { pub enum SearchKind { /// Joint candidate-connection planning, the facade's search. CandidateConnections, - /// Every generated candidate (`--exhaustive`), up to - /// [`EXHAUSTIVE_MAX_COMBINATIONS`]. - Exhaustive, } -/// `--exhaustive`'s default evaluated-assignment budget. -pub const EXHAUSTIVE_MAX_COMBINATIONS: usize = 10_000; - /// How the tester plans and judges. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct FuzzSettings { @@ -368,7 +353,7 @@ pub struct Planned { } /// A plan priced by Stage 2 and Stage 3 outside the search: the cheapest -/// materialization assignment of given logical roots. +/// physical properties assignment of given logical roots. #[derive(Debug, Clone, PartialEq, Serialize)] pub struct Priced { pub selected: String, @@ -443,70 +428,21 @@ impl Planner { .with_deployment_capabilities(&deployment) .with_ignore_accuracy_guarantees(!self.settings.enforce_accuracy_guarantees); let run = match self.settings.search { - SearchKind::Exhaustive => run_planning_stages_over( - roots.clone(), - &entries, - &data, - Some(EXHAUSTIVE_MAX_COMBINATIONS), - models, - ), SearchKind::CandidateConnections => run_search_over( roots.clone(), &entries, &data, models, PlanSearchSettings::default(), - None, ), } .map_err(|e| e.to_string())?; - match &run.reference { - // The reference pricer's selection among the enumerated plans. - Some(reference) => { - let selection = reference - .selection - .as_ref() - .map_err(|e| format!("selection: {e}"))?; - let cost_per_second = selection.costs[&selection.selected] - .cost_per_second - .ok_or("no cost per second (cost model 3)")?; - let from_logical = reference - .physical - .iter() - .find(|p| p.id == selection.selected) - .map(|p| p.from_logical.clone()); - let logical_roots = from_logical.and_then(|id| { - reference - .choices - .iter() - .find(|c| c.id == id) - .and_then(|c| c.roots.as_ref().ok().cloned()) - }); - Ok(Planned { - selected: selection.selected.clone(), - cost_per_second, - stage0_roots: roots, - logical_roots, - }) - } - None => { - let selection = run - .run - .selection - .as_ref() - .map_err(|e| format!("selection: {e}"))?; - let cost_per_second = selection - .cost - .cost_per_second - .ok_or("no cost per second (cost model 3)")?; - Ok(Planned { - selected: "selected".into(), - cost_per_second, - stage0_roots: roots, - logical_roots: run.run.selected_plan.clone(), - }) - } - } + let (_plan, _selection) = run + .run + .selection + .as_ref() + .map_err(|e| format!("selection: {e}"))?; + todo!("#1207 PR 9: the selected plan's cost (plan_cost) and roots from its PhysicalASAPDAG") } /// Stage 2 and Stage 3 over the logical plan `roots` of `workload`'s @@ -514,7 +450,7 @@ impl Planner { /// assignment implemented and priced, the cheapest selected, with the /// same models as [`plan`](Self::plan). Guarantees are composed again /// first, so a derived node carries the guarantee of what it reads, and - /// summaries are sized for their readers as Pass 1 sizes them. + /// summaries are sized for their consumers as Pass 1 sizes them. /// With `last_root_is_key_filter`, the last root is a `Filter` on /// group-key columns only and keeps its input's guarantee (whole groups /// selected by an exact key keep each remaining row's bound), which @@ -526,93 +462,8 @@ impl Planner { roots: &[QueryRoot], last_root_is_key_filter: bool, ) -> Result { - for root in roots { - root.validate_structure() - .map_err(|e| format!("witness validation: {e}"))?; - } - let mut roots = asap_accuracy::propagation::recompose_guarantees(roots) - .map_err(|e| format!("witness guarantees: {e}"))?; - let planning = workload.planning_workload(); - let entries: Vec<_> = planning.query_workload.entries().collect(); - // A witness is built outside the memo: size its summaries as Pass 1 - // sizes a candidate for these readers, which Stage 3 only checks. - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - roots = asap_accuracy::sizing::with_summaries_sized_for_targets( - &roots, - &targets, - &workload.data, - ) - .map_err(|e| format!("witness sizing: {e}"))?; - if let (true, Some(last)) = (last_root_is_key_filter, roots.last_mut()) { - let filter = last.as_operator().ok_or("a scalar root")?; - let input = filter.operator.children()[0].guarantee.clone(); - *last = QueryRoot::Operator(Rc::new((**filter).clone().with_guarantee(input))); - } - let cost_inputs = self.cost_inputs(); - let deployment = workload.deployment(); - let models = PlanningModels::builtin() - .with_cost_inputs(&cost_inputs) - .with_deployment_capabilities(&deployment) - .with_ignore_accuracy_guarantees(!self.settings.enforce_accuracy_guarantees); - let mut candidates = optimize_physical_asap( - "witness", - &roots, - &entries, - &workload.data, - deployment.query_time_results_retained, - ) - .map_err(|e| format!("witness Stage 2: {e}"))?; - for (index, candidate) in candidates.iter_mut().enumerate() { - candidate.id = format!("witness-m{index}"); - } - let selection = - crate::memo_pricing::reference_select(&candidates, &entries, &workload.data, models) - .map_err(|e| format!("witness Stage 3: {e}"))?; - let cost = &selection.costs[&selection.selected]; - let cost_per_second = cost - .cost_per_second - .ok_or("no cost per second (cost model 3)")?; - let candidate = candidates - .iter() - .find(|c| c.id == selection.selected) - .expect("the selected candidate is offered"); - let dag = &candidate.dag; - let mut inputs: BTreeMap<_, Vec<_>> = BTreeMap::new(); - for edge in &dag.edges { - inputs.entry(edge.consumer).or_default().push(edge.producer); - } - let reached = |root| { - let mut seen = BTreeSet::new(); - let mut stack = vec![root]; - while let Some(node) = stack.pop() { - if seen.insert(node) { - stack.extend(inputs.get(&node).into_iter().flatten().copied()); - } - } - seen - }; - let reached_by_root: Vec> = dag.roots.iter().map(|r| reached(*r)).collect(); - let (last, others) = reached_by_root.split_last().ok_or("no roots")?; - let private: Vec<_> = last - .iter() - .filter(|node| others.iter().all(|set| !set.contains(node))) - .filter_map(|node| cost.per_node.get(node)) - .collect(); - let derived_root = PrivateNodes { - cost_per_second: private.iter().filter_map(|n| n.cost_per_second).sum(), - operators: private - .iter() - .map(|n| format!("{:?}", n.operator)) - .collect(), - }; - Ok(Priced { - selected: selection.selected.clone(), - cost_per_second, - derived_root, - }) + let _ = (workload, roots, last_root_is_key_filter); + todo!("#1207 PR 6: price a witness plan through the new Stage 2 and Stage 3") } /// [`price`](Self::price) of `workload`'s selected logical plan diff --git a/crates/devtools/src/planner_fuzz/relations.rs b/crates/devtools/src/planner_fuzz/relations.rs index 1b72e8cc9..676b5805e 100644 --- a/crates/devtools/src/planner_fuzz/relations.rs +++ b/crates/devtools/src/planner_fuzz/relations.rs @@ -15,15 +15,15 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::operator::operator_properties::Reduction; +use asap_types::ir::operator::arguments::Reduction; use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::family_capability::FamilyCapability; -use asap_types::ir::schema::sketch::{SketchAlgorithm, SketchStatistic}; use asap_types::ir::schema::{DataType, FieldDataType}; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ProjectItem, QueryRoot, ScalarExpr, }; +use asap_types::summary_type::SummaryTypeCapability; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; use super::generate::{ label_value, looser, tighter, OuterAggregation, OuterOp, PromqlSpec, QuerySpec, RangeFunction, @@ -987,11 +987,18 @@ fn with_root_estimate( } /// The family of the build whose state `state` carries. -fn state_family(state: &OperatorNode) -> Result { +fn state_family(state: &Rc) -> Result { match state.summary_build() { Some( - ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => Ok(family.clone()), + ASAPOp::SummaryAgg { + summary_type: family, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + summary_type: family, + .. + }, + ) => Ok(family), _ => Err("q's estimate reads no single summary build".into()), } } @@ -1118,7 +1125,7 @@ pub fn witness_root( Witness::SketchRollup { labels } => { let (state, query) = root_estimate(q_root)?; let family = state_family(&state)?; - if !FamilyCapability::merges_groups(&family) { + if !SummaryTypeCapability::merges_groups(&family) { return Err(format!("q's family {family:?} does not merge groups")); } let group_by = labels @@ -1142,7 +1149,6 @@ pub fn witness_root( .collect(); let merge = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![state], - newest_tumbling_windows: None, group_by: Some(group_by), })) .map_err(|e| format!("rollup merge: {e}"))?; @@ -1153,45 +1159,10 @@ pub fn witness_root( .map_err(|e| format!("rollup estimate: {e}"))?; with_root_estimate(q_root, estimate, &dropped) } - Witness::NewestTumblingWindows { window_ms } => rewrite_root( - q_root, - "summary merge over a tumbling-window chain", - |node| { - let Some(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: None, - group_by, - }) = node.asap() - else { - return None; - }; - let [chain] = children.as_slice() else { - return None; - }; - let Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) = chain.asap() - else { - return None; - }; - let tumbling_windows = window_ms / tumbling_window_width_ms; - (window_ms % tumbling_window_width_ms == 0 - && tumbling_windows >= 1 - && (tumbling_windows as u32) < *tumbling_window_count) - .then(|| { - with_operator( - node, - Operator::ASAP(ASAPOp::SummaryMerge { - children: children.clone(), - newest_tumbling_windows: Some(tumbling_windows as u32), - group_by: group_by.clone(), - }), - ) - }) - }, - ), + Witness::NewestTumblingWindows { window_ms } => { + let _ = window_ms; + todo!("#1207 PR 9: a witness merging the windows a TimeRange over q's chain selects") + } Witness::QuantileRank { from, to } => { let mut done = false; rewrite_root(q_root, "quantile estimate", |node| { @@ -1226,9 +1197,9 @@ pub fn witness_root( }) } Witness::OtherStatistic { statistic } => { - let answers = |state: &OperatorNode| -> Result<(), String> { + let answers = |state: &Rc| -> Result<(), String> { let family = state_family(state)?; - if FamilyCapability::of(&family).answers(statistic) { + if SummaryTypeCapability::of(&family).answers(statistic) { Ok(()) } else { Err(format!( @@ -1278,7 +1249,7 @@ pub fn witness_root( return Err("both selections read one state".into()); } let family = state_family(&state)?; - if FamilyCapability::of(&family).merge_inputs.is_none() { + if SummaryTypeCapability::of(&family).merge_inputs.is_none() { return Err(format!("q's family {family:?} has no merge")); } // The merge's coverage check refuses states of different @@ -1286,7 +1257,6 @@ pub fn witness_root( // selections. let merge = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![state, other_state], - newest_tumbling_windows: None, group_by: None, })) .map_err(|e| format!("union merge: {e}"))?; @@ -1421,130 +1391,15 @@ pub fn witness_roots( Ok(roots) } -/// The first node under `node` (depth first, `node` included) that `pred` -/// accepts. -fn find_first( - node: &Rc, - pred: &dyn Fn(&OperatorNode) -> bool, -) -> Option> { - if pred(node) { - return Some(node.clone()); - } - node.operator - .children() - .into_iter() - .find_map(|child| find_first(child, pred)) -} - -/// The distinct nodes under `node` (`node` included) that `pred` accepts. -fn collect( - node: &Rc, - pred: &dyn Fn(&OperatorNode) -> bool, - found: &mut Vec>, -) { - if found.iter().any(|f| Rc::ptr_eq(f, node)) { - return; - } - if pred(node) { - found.push(node.clone()); - } - for child in node.operator.children() { - collect(child, pred, found); - } -} - -/// `W`'s roots with the tumbling-window chain `q` merges whole lengthened to cover -/// `window_ms`, every merge of it reading its old newest tumbling windows of the -/// longer chain; and `q`'s root merging the whole longer chain. +/// `W`'s roots with the reading of `q`'s tumbling-window chain lengthened +/// to cover `window_ms`, and `q`'s root reading the longer window. fn longer_chain_roots( base_roots: &[QueryRoot], target: usize, window_ms: u64, ) -> Result<(Vec, Rc), String> { - let q_root = base_roots[target] - .as_operator() - .ok_or("q is a scalar query")?; - let whole_chain_merge = |node: &OperatorNode| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { children, newest_tumbling_windows: None, .. }) - if matches!(children.as_slice(), [chain] if matches!(chain.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - ) - }; - let merge = find_first(q_root, &whole_chain_merge) - .ok_or("q's plan has no summary merge over a tumbling-window chain")?; - // Lengthening one of several chains (say, an SQL idiom's exact row - // count beside its statistic) would leave the others at q's window. - let mut merges = Vec::new(); - collect(q_root, &whole_chain_merge, &mut merges); - if merges.len() > 1 { - return Err(format!( - "q's plan merges {} tumbling-window chains whole", - merges.len() - )); - } - let Some(ASAPOp::SummaryMerge { children, .. }) = merge.asap() else { - unreachable!("found a merge"); - }; - let chain = children[0].clone(); - let mut operator = chain.operator.clone(); - let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) = &mut operator - else { - unreachable!("found a tumbling-window chain"); - }; - let old_count = *tumbling_window_count; - if !window_ms.is_multiple_of(*tumbling_window_width_ms) - || window_ms / *tumbling_window_width_ms <= old_count as u64 - { - return Err(format!( - "q's chain of {old_count} tumbling windows of {tumbling_window_width_ms} ms does not extend to {window_ms} ms" - )); - } - *tumbling_window_count = (window_ms / *tumbling_window_width_ms) as u32; - let long_chain = with_operator(&chain, operator); - let reread = |node: &Rc, whole: bool| { - let Some(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows, - group_by, - }) = node.asap() - else { - return None; - }; - let [child] = children.as_slice() else { - return None; - }; - Rc::ptr_eq(child, &chain).then(|| { - with_operator( - node, - Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![long_chain.clone()], - newest_tumbling_windows: if whole { - None - } else { - Some(newest_tumbling_windows.unwrap_or(old_count)) - }, - group_by: group_by.clone(), - }), - ) - }) - }; - let mut cache = HashMap::new(); - let roots = base_roots - .iter() - .map(|root| match root.as_operator() { - Some(node) => { - rewrite(node, &mut |n| reread(n, false), &mut cache).map(QueryRoot::Operator) - } - None => Ok(root.clone()), - }) - .collect::, _>>()?; - let long_root = rewrite(q_root, &mut |n| reread(n, true), &mut HashMap::new())?; - Ok((roots, long_root)) + let _ = (base_roots, target, window_ms); + todo!("#1207 PR 9: lengthen the TimeRange over q's chain, which has no window count") } fn planned( @@ -1908,16 +1763,20 @@ pub fn disjoint_copy(workload: &FuzzWorkload) -> Option { both.data.sources.extend(copy.data.sources); both.tables.extend(copy.tables); both.data.arrival = match workload.data.arrival { - asap_types::workload::DataArrival::Ingesting { rows_per_second } => { - asap_types::workload::DataArrival::Ingesting { - rows_per_second: 2.0 * rows_per_second, - } - } - asap_types::workload::DataArrival::Mixed { rows_per_second } => { - asap_types::workload::DataArrival::Mixed { - rows_per_second: 2.0 * rows_per_second, - } - } + asap_types::workload::DataArrival::Ingesting { + rows_per_second, + max_out_of_order_ms, + } => asap_types::workload::DataArrival::Ingesting { + rows_per_second: 2.0 * rows_per_second, + max_out_of_order_ms, + }, + asap_types::workload::DataArrival::Mixed { + rows_per_second, + max_out_of_order_ms, + } => asap_types::workload::DataArrival::Mixed { + rows_per_second: 2.0 * rows_per_second, + max_out_of_order_ms, + }, other => other, }; Some(both) @@ -1927,7 +1786,7 @@ pub fn disjoint_copy(workload: &FuzzWorkload) -> Option { mod tests { use super::*; use crate::planner_fuzz::generate::{generated_workload, SqlSpec}; - use asap_types::ir::properties::AccuracyTarget; + use asap_types::accuracy::AccuracyTarget; fn promql(spec: PromqlSpec, accuracy: AccuracyTarget) -> FuzzWorkload { generated_workload( diff --git a/crates/devtools/src/planner_fuzz/report.rs b/crates/devtools/src/planner_fuzz/report.rs index 330c8de94..703543743 100644 --- a/crates/devtools/src/planner_fuzz/report.rs +++ b/crates/devtools/src/planner_fuzz/report.rs @@ -81,7 +81,7 @@ pub fn documented_patterns() -> Vec { note: "the operator follows q's own: topk, bottomk, max, quantile(0.5) or a threshold `> 0` (ASAPWorkloads #4, #12, #17, #18)", }, DocumentedPattern { - pattern: "Summary capability: another statistic the family answers from one build (FamilyCapability::answers)", + pattern: "Summary capability: another statistic the family answers from one build (SummaryTypeCapability::answers)", doc_reference: "planner-layering.md Pass 2 rule table L586, examples L611–617, Example 2 L1306–1370", relation: Some(Relation::OtherStatistic), note: "UnivMon's count, distinct count, L2 and entropy (ASAPWorkloads #57, #107–#108); the witness reads q's own state, or is W ∪ {q'} planned with UnivMon as the only family", diff --git a/crates/devtools/src/planner_fuzz/shrink.rs b/crates/devtools/src/planner_fuzz/shrink.rs index afcdf98fa..f64b2118d 100644 --- a/crates/devtools/src/planner_fuzz/shrink.rs +++ b/crates/devtools/src/planner_fuzz/shrink.rs @@ -1,7 +1,7 @@ //! Greedy shrinking of a violated check: drop queries and simplify //! expressions while the violation still reproduces. -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use super::generate::{ OuterOp, QuerySpec, RangeFunction, SqlAggregate, COUNTER_METRIC, GAUGE_METRIC, diff --git a/crates/devtools/src/planner_layering_examples.rs b/crates/devtools/src/planner_layering_examples.rs index f4ac75793..f4f2f09f2 100644 --- a/crates/devtools/src/planner_layering_examples.rs +++ b/crates/devtools/src/planner_layering_examples.rs @@ -2,7 +2,7 @@ //! (`docs/design_docs/proposals/planner-layering.md`), shared by //! `tools/dag-viewer/workloads/planner-layering/` and the planner-layering acceptance tests. -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, PrometheusMetricType, Query, QueryLanguage, QueryRequirements, QueryWorkload, RepeatedDemand, @@ -72,6 +72,7 @@ pub fn example1() -> PlanningWorkload { let mut data = shared_data_workload( DataArrival::Ingesting { rows_per_second: SHARED_ROWS_PER_SECOND, + max_out_of_order_ms: None, }, EXAMPLE1_METRIC, ); @@ -141,6 +142,7 @@ pub fn example2() -> (PlanningWorkload, asap_frontend_sql::SqlCatalog) { data_workload: Some(DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1_000_000.0 / 15.0, + max_out_of_order_ms: None, }, sources: [( "flows".to_string(), @@ -305,6 +307,7 @@ pub fn pattern_b() -> PlanningWorkload { data_workload: Some(shared_data_workload( DataArrival::Ingesting { rows_per_second: SHARED_ROWS_PER_SECOND, + max_out_of_order_ms: None, }, PATTERN_METRIC, )), @@ -322,9 +325,11 @@ mod tests { DataArrival::AtRest, DataArrival::Ingesting { rows_per_second: SHARED_ROWS_PER_SECOND, + max_out_of_order_ms: None, }, DataArrival::Mixed { rows_per_second: SHARED_ROWS_PER_SECOND, + max_out_of_order_ms: None, }, ] { shared_data_workload(arrival, PATTERN_METRIC) diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index b40f36dea..56967913c 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -4,32 +4,20 @@ use asap_frontend_sql::SqlCatalog; use asap_logical_asap_optimization::{ - generate_dag_of_logical_memo_groups, generate_per_computation_logical_memo_candidates, - pass2_after_local_window_composition, pass2_local_window_composition, - PerComputationLogicalMemoCandidatesError, -}; -use asap_physical_asap_optimization::{ - generate_physical_asap_memo_candidates, Stage2AssignmentRealizer, + memo_of_queries, pass1_memo, pass2_after_local_window_composition, + pass2_local_window_composition, Pass1Error, }; +use asap_physical_asap_optimization::generate_physical_asap_memo_candidates; use asap_plan_selection::select_plan; use asap_planner::{ run_planning, PlanSearchSettings, PlanningModels, PlanningReport, PlanningRun, SelectionError, }; -use asap_types::ir::schema::schema_support::with_promql_series_identity; +use asap_types::ir::schema::series_identity::with_promql_series_identity; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::QueryRoot; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::stages::{AssignmentRealizer, DagOfLogicalMemoGroups, Memo}; -use asap_types::workload::bounds::with_confirmed_weights_at; +use asap_types::stages::Memo; use asap_types::workload::{DataWorkload, PlanningWorkload, QueryWorkloadEntry}; use serde_json::Value; -use std::collections::HashSet; -use std::rc::Rc; - -use crate::memo_pricing::{reference_enumeration, ReferenceEnumeration}; - -/// The memo choices the reference enumeration prices when no limit is given. -pub const DEFAULT_REFERENCE_CHOICES: usize = 10_000; /// The last stage a run runs (`asap_planner --run-until`); the run's /// fields for later stages are left empty. @@ -87,63 +75,6 @@ pub struct PlanningStagesRun { pub pass2_local_window_composition: Option, /// The stages' outputs: Pass 1, Pass 2, Stage 2 and the selection. pub run: PlanningRun, - /// With a reference enumeration: Pass 2's memo choices, each implemented - /// by Stage 2 and all priced by the reference pricer. - pub reference: Option, -} - -/// A logical candidate the document shows: its id and composed roots, or -/// why they do not compose. -#[derive(Debug, Clone)] -pub struct ShownLogicalCandidate { - pub id: String, - pub roots: Result, String>, -} - -impl PlanningStagesRun { - /// The logical candidates the document shows: the selected plan (`L1`; - /// its roots as Stage 2 built them, with Stage 3's weight proofs), then - /// with a reference enumeration, its's choices (`E`). - pub fn logical_candidates(&self) -> Vec { - let selected = self - .run - .selected_plan - .iter() - .map(|roots| ShownLogicalCandidate { - id: "L1".into(), - roots: Ok(roots.clone()), - }); - let enumerated = self.reference.iter().flat_map(|reference| { - reference - .choices - .iter() - .map(|choice| ShownLogicalCandidate { - id: choice.id.clone(), - roots: choice.roots.clone(), - }) - }); - selected.chain(enumerated).collect() - } - - /// The physical candidates the document shows: the selected plan as - /// Stage 2 built it (`P1`, from `L1`), then with a reference enumeration every - /// plan the reference pricer priced (`E-`). - pub fn physical_candidates(&self) -> Vec { - let selected = - self.run - .selected_physical_plan - .iter() - .map(|realized| PhysicalWorkloadPlan { - id: "P1".into(), - from_logical: "L1".into(), - ..realized.plan.clone() - }); - let enumerated = self - .reference - .iter() - .flat_map(|reference| reference.physical.iter().cloned()); - selected.chain(enumerated).collect() - } } /// Lower a PromQL workload, each series' identity as a column (the row @@ -183,7 +114,6 @@ pub fn lower_sql_roots( &entry.query.0, catalog, dialect.clone(), - entry.requirements.accuracy.clone(), )) .map(QueryRoot::Operator) .map_err(|e| format!("lowering: {e}")) @@ -215,23 +145,14 @@ impl From for String { } } -/// The planning stages over `workload`, with the reference enumeration of -/// at most `max_combinations` memo choices (default -/// [`DEFAULT_REFERENCE_CHOICES`]). +/// The planning stages over `workload`. pub fn run_planning_stages( workload: &PlanningWorkload, roots: Vec, - max_combinations: Option, ) -> Result { let entries: Vec<_> = workload.query_workload.entries().collect(); let data = workload.data_workload.clone().unwrap_or_default(); - run_planning_stages_over( - roots, - &entries, - &data, - max_combinations, - PlanningModels::builtin(), - ) + run_planning_stages_over(roots, &entries, &data, PlanningModels::builtin()) } /// [`run_planning_stages`] over roots lowered from any mix of query @@ -244,7 +165,6 @@ pub fn run_planning_stages_over( roots: Vec, entries: &[QueryWorkloadEntry], data: &DataWorkload, - max_combinations: Option, models: PlanningModels<'_>, ) -> Result { run_search_over( @@ -252,14 +172,11 @@ pub fn run_planning_stages_over( entries, data, models, - // The reference enumeration reads Pass 2's memo. PlanSearchSettings { inspect: true }, - Some(max_combinations.unwrap_or(DEFAULT_REFERENCE_CHOICES)), ) } -/// The facade's planning, as `plan_installable` runs it, without a -/// reference enumeration. Use the executor's summary capabilities in +/// The facade's planning, as `plan_installable` runs it. Use the executor's summary capabilities in /// `models` when comparing with installation behavior. pub fn run_default_search_over( roots: Vec, @@ -267,53 +184,29 @@ pub fn run_default_search_over( data: &DataWorkload, models: PlanningModels<'_>, ) -> Result { - run_search_over( - roots, - entries, - data, - models, - PlanSearchSettings::default(), - None, - ) + run_search_over(roots, entries, data, models, PlanSearchSettings::default()) } -/// The facade's planning with `settings`, and with `reference_choices` -/// the reference enumeration of that many memo choices. +/// The facade's planning with `settings`. pub fn run_search_over( roots: Vec, entries: &[QueryWorkloadEntry], data: &DataWorkload, models: PlanningModels<'_>, settings: PlanSearchSettings, - reference_choices: Option, ) -> Result { validate(&roots)?; let run = run_planning(roots.clone(), entries, data, models, settings) .map_err(|e| PlanningStagesError::Pass1(e.to_string()))?; - if let Some(roots) = &run.selected_plan { - validate(roots)?; - } - let reference = reference_choices - .and_then(|max| reference_enumeration(&run, &roots, entries, data, models, max)); Ok(PlanningStagesRun { run_until: RunUntil::Stage3, stage0_memo_groups: None, pass2_local_window_composition: None, stage0_query_language_frontends: roots, run, - reference, }) } -/// Stage 0's memo groups, `dag`, without candidates. -fn stage0_memo_groups(dag: DagOfLogicalMemoGroups) -> Memo { - Memo { - candidates: vec![Vec::new(); dag.groups.len()], - groups: dag.groups, - query_roots: dag.query_roots, - } -} - /// The stages over `roots` up to and including `until`, as the facade runs /// them. A stage's failure is reported in the run, as the facade does. // ponytail: copies run_planning's stage order, which has no stop point; @@ -326,38 +219,35 @@ pub fn run_stages_until( until: RunUntil, ) -> Result { validate(&roots)?; - let stage0_dag = generate_dag_of_logical_memo_groups(&roots); + // As `run_planning`: each query with its accuracy target, or none when + // accuracy is not enforced. + let accuracy_targets: Vec<_> = entries + .iter() + .map(|entry| { + (!models.ignore_accuracy_guarantees).then(|| entry.requirements.accuracy.clone()) + }) + .collect(); + let stage0_dag = memo_of_queries(&roots, &accuracy_targets, data) + .map_err(|e| PlanningStagesError::Validation(e.to_string()))?; let mut stages = PlanningStagesRun { run_until: until, - stage0_memo_groups: Some(stage0_memo_groups(stage0_dag.clone())), + stage0_memo_groups: Some(stage0_dag.clone()), pass2_local_window_composition: None, stage0_query_language_frontends: roots.clone(), run: PlanningRun { - per_computation_candidates: None, + pass1_memo: None, logical_asap_candidates: None, physical_asap_candidates: None, selection: Err(SelectionError::NotRun), - selected_physical_plan: None, - selected_plan: None, report: PlanningReport::default(), }, - reference: None, }; if until == RunUntil::Stage0 { return Ok(stages); } let run = &mut stages.run; - let accuracy_targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let accuracy = models.accuracy_check_inputs(); - let pass1 = generate_per_computation_logical_memo_candidates( + let pass1 = pass1_memo( stage0_dag, - &accuracy_targets, - accuracy.model, - accuracy.evidence, - data, models .deployment_capabilities .supported_summaries @@ -365,7 +255,7 @@ pub fn run_stages_until( ); // As `run_planning`: a query Pass 1 cannot plan is reported before // contradictory deployment capabilities. - if let Err(PerComputationLogicalMemoCandidatesError::PerComputation(error)) = &pass1 { + if let Err(Pass1Error::PerComputation(error)) = &pass1 { return Err(PlanningStagesError::Pass1(error.to_string())); } if let Err(error) = models.deployment_capabilities.validate(data) { @@ -379,13 +269,19 @@ pub fn run_stages_until( return Ok(stages); } }; - run.per_computation_candidates = Some(pass1.clone()); + run.pass1_memo = Some(pass1.clone()); if until == RunUntil::Pass1 { return Ok(stages); } - let composed = pass2_local_window_composition(pass1, entries); + let composed = match pass2_local_window_composition(pass1, entries) { + Ok(composed) => composed, + Err(error) => { + run.selection = Err(SelectionError::LogicalASAPCandidates(error.to_string())); + return Ok(stages); + } + }; stages.pass2_local_window_composition = Some(composed.clone()); - let logical = match pass2_after_local_window_composition(composed) { + let logical = match pass2_after_local_window_composition(composed, models.evidence) { Ok(logical) => logical, Err(error) => { run.selection = Err(SelectionError::LogicalASAPCandidates(error.to_string())); @@ -398,61 +294,14 @@ pub fn run_stages_until( return Ok(stages); } let logical = run.logical_asap_candidates.as_ref().expect("set above"); - let physical = match generate_physical_asap_memo_candidates( - logical, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - models.deployment_capabilities.raw_data_retained, - ) { - Ok(physical) => physical, - Err(error) => { - run.selection = Err(SelectionError::PhysicalASAPCandidates(error.to_string())); - return Ok(stages); - } - }; + let physical = + generate_physical_asap_memo_candidates(logical, data, models.deployment_capabilities); if until == RunUntil::Stage2 { run.physical_asap_candidates = Some(physical); return Ok(stages); } - let realizer = Stage2AssignmentRealizer { - original_roots: &roots, - entries, - data, - query_time_results_retained: models.deployment_capabilities.query_time_results_retained, - }; - match select_plan( - &physical, - entries, - data, - &accuracy, - models.cost_inputs, - &realizer, - ) { - Ok(selection) => { - match realizer.realize(&physical, &selection.selected_dag) { - Ok(realized) => { - // As the facade does: the plan to run carries the weight - // proofs Stage 3 made. - let proven: HashSet<*const asap_types::ir::OperatorNode> = selection - .nodes_with_proven_non_negative_weights - .iter() - .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) - .map(Rc::as_ptr) - .collect(); - let plan = with_confirmed_weights_at(&realized.plan.roots, data, &|node| { - proven.contains(&Rc::as_ptr(node)) - }); - validate(&plan)?; - run.selected_plan = Some(plan); - run.selected_physical_plan = Some(realized); - run.selection = Ok(selection); - } - Err(error) => run.selection = Err(SelectionError::Realization(error)), - } - } - Err(error) => run.selection = Err(SelectionError::Selection(error)), - } + run.selection = select_plan(&physical, entries, data, models.cost_inputs) + .map_err(SelectionError::Selection); run.physical_asap_candidates = Some(physical); Ok(stages) } diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index f68332018..1f89f597a 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -1,6 +1,6 @@ //! The planner stages' output as plain text: `asap_planner --format text`. //! It prints, up to the run's last stage, the frontends' roots and their -//! memo groups (Stage 0), Pass 1's `PerComputationLogicalMemoCandidates`, +//! memo groups (Stage 0), Pass 1's `Memo`, //! the memo after Pass 2's local window composition and Pass 2's //! `LogicalASAPMemoCandidates` (Stage 2's input), Stage 2's //! `PhysicalASAPMemoCandidates` (Stage 3's input), and Stage 3's @@ -17,17 +17,14 @@ //! G1: TimeRange 60s <- G0 //! G2: Aggregate rate() <- G1 //! [0] Pass1 FinalizeExactAccumulator ← SummaryAgg inputs=[G1] -//! [1] TumblingWindows new FinalizeExactAccumulator ← SummaryMerge ← SummaryTumblingWindowChain tumbling_window_count=6 tumbling_window_width_ms=10000 inputs=[G0] +//! [1] TumblingWindows new FinalizeExactAccumulator ← SummaryMerge ← SummaryTumblingWindowChain tumbling_window_width_ms=10000 inputs=[G0] //! ``` -use std::collections::HashSet; use std::fmt::Write; -use std::rc::Rc; -use asap_planner::selected_chosen_candidates; use asap_types::ir::flat::flatten; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{Memo, MemoGroupId, PhysicalASAPMemoCandidates}; +use asap_types::ir::QueryRoot; +use asap_types::stages::{Memo, MemoGroup, PhysicalASAPMemoCandidates}; use crate::candidate_labels::node_label; use crate::planning_stages::{PlanningStagesRun, RunUntil}; @@ -52,7 +49,7 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { write_dag(&mut out, &stages.stage0_query_language_frontends); section( &mut out, - "Stage 0 → Stage 1: DagOfLogicalMemoGroups (the roots hash-consed)", + "Stage 0 → Stage 1: Memo (the roots hash-consed, without candidates)", ); if let Some(memo) = &stages.stage0_memo_groups { write_memo(&mut out, memo, None, false); @@ -61,32 +58,12 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { return out; } - let Some(pass1) = &run.per_computation_candidates else { + let Some(pass1) = &run.pass1_memo else { failure(&mut out); return out; }; - section( - &mut out, - "Stage 1 Pass 1 output: PerComputationLogicalMemoCandidates", - ); - write_memo(&mut out, &pass1.memo, None, true); - let _ = writeln!( - out, - "pruned_candidates ({}):", - pass1.pruned_candidates.len() - ); - let rows = pass1 - .pruned_candidates - .iter() - .map(|pruned| { - vec![ - format!("G{}:", pruned.group.0), - format!("{:?}", pruned.candidate), - format!("{:?}", pruned.reason), - ] - }) - .collect(); - write_rows(&mut out, " ", rows); + section(&mut out, "Stage 1 Pass 1 output: Memo"); + write_memo(&mut out, pass1, None, true); if until == RunUntil::Pass1 { return out; } @@ -102,7 +79,7 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { &mut out, "Stage 1 Pass 2, after local window composition (tumbling-window candidates)", ); - write_memo(&mut out, pass2, Some(&pass1.memo), true); + write_memo(&mut out, pass2, Some(pass1), true); section( &mut out, "Stage 1 Pass 2 output = Stage 2 input: LogicalASAPMemoCandidates", @@ -126,20 +103,6 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { } section(&mut out, "Stage 3 output: Selection"); - let chosen = match &run.selection { - Ok(selection) => { - selected_chosen_candidates(&logical.memo, physical, &selection.selected_dag) - } - Err(_) => Err("no plan was selected".into()), - }; - match &chosen { - Ok(choice) => { - let _ = writeln!(out, "selected choice: {}", choice_text(choice)); - } - Err(reason) => { - let _ = writeln!(out, "selected choice unavailable: {reason}"); - } - } write_selection(&mut out, stages); out } @@ -215,47 +178,24 @@ fn reads(inputs: &[String]) -> String { } } -/// `G0, G1`. -fn group_list(groups: &[MemoGroupId]) -> String { - groups - .iter() - .map(|group| format!("G{}", group.0)) - .collect::>() - .join(", ") -} - -/// `G2:c0 G3:c0`: each chosen group's candidate. -fn choice_text(choice: &asap_logical_asap_optimization::ChosenCandidatePerMemoGroup) -> String { - choice - .iter() - .map(|(group, candidate)| format!("G{}:c{candidate}", group.0)) - .collect::>() - .join(" ") -} - /// `memo`'s groups, one line each with the child groups it reads, and with -/// `with_candidates` each group's candidates below it: index, origin, -/// `new` when `previous` (the memo at the step before) has no candidate of -/// that group with the same node and inputs, chain and input groups (fused -/// groups, and accuracy budget parts when more than one). +/// `with_candidates` each group's candidates below it, marked `new` when +/// `previous` (the memo at the step before) has no such candidate. fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candidates: bool) { // The group lines share one set of column widths, with each group's // candidates printed under its line. let group_rows: Vec> = memo - .groups + .groups() .iter() .enumerate() .map(|(id, group)| { let children: Vec = group - .child_groups - .iter() + .expression + .children() + .into_iter() .map(|child| format!("G{}", child.0)) .collect(); - vec![ - format!("G{id}:"), - root_label(&group.logical_node), - reads(&children), - ] + vec![format!("G{id}:"), group_label(group), reads(&children)] }) .collect(); let mut group_lines = String::new(); @@ -265,48 +205,13 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi if !with_candidates { continue; } - let earlier = previous - .and_then(|memo| memo.candidates.get(id)) - .map_or(&[][..], Vec::as_slice); - let rows = memo.candidates[id] - .iter() - .enumerate() - .map(|(index, candidate)| { - let added = previous.is_some() - && !earlier.iter().any(|before| { - Rc::ptr_eq(&before.root_node, &candidate.root_node) - && before.input_groups == candidate.input_groups - }); - let mut inputs = format!("inputs=[{}]", group_list(&candidate.input_groups)); - if !candidate.fused_groups.is_empty() { - let _ = write!(inputs, " fuses=[{}]", group_list(&candidate.fused_groups)); - } - if candidate.accuracy_budget_parts != 1 { - let _ = write!( - inputs, - " accuracy_budget_parts={}", - candidate.accuracy_budget_parts - ); - } - vec![ - format!("[{index}]"), - format!("{:?}", candidate.origin), - if added { - "new".to_owned() - } else { - String::new() - }, - chain_label(memo, &candidate.root_node, &candidate.input_groups), - inputs, - ] - }) - .collect(); - write_rows(out, " ", rows); + let _ = (previous, &memo.groups()[id].candidates); + todo!("#1207 PR 9: print each candidate's expression, logical properties and input requirements") } let roots: Vec = memo - .query_roots + .queries .iter() - .map(|root| match root { + .map(|query| match &query.root { QueryRoot::Operator(group) => format!("G{}", group.0), scalar => format!("{scalar:?}"), }) @@ -315,112 +220,8 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi } fn write_physical(out: &mut String, physical: &PhysicalASAPMemoCandidates) { - let nodes = &physical.operator_nodes_of_candidates_for_memo_groups; - let _ = writeln!( - out, - "operator_nodes_of_candidates_for_memo_groups ({}):", - nodes.len() - ); - let rows = nodes - .iter() - .enumerate() - .map(|(id, node)| { - let implements: Vec = node - .candidates_for_memo_group - .iter() - .map(|candidate| format!("G{}[{}]", candidate.group.0, candidate.index)) - .collect(); - let mut role = if node.is_implementation_root { - "implementation root".to_owned() - } else { - "private".to_owned() - }; - if node.accuracy_budget_parts != 1 { - let _ = write!( - role, - " accuracy_budget_parts={}", - node.accuracy_budget_parts - ); - } - vec![ - format!("O{id}:"), - root_label(&node.physical_node), - format!("implements=[{}]", implements.join(", ")), - role, - ] - }) - .collect(); - write_rows(out, " ", rows); - - let _ = writeln!(out, "candidates ({}):", physical.candidates.len()); - let rows = physical - .candidates - .iter() - .enumerate() - .map(|(id, candidate)| { - // One `{...}` per input: the candidates that may produce it. - let producers: Vec = candidate - .producer_candidates_per_input - .iter() - .map(|producers| { - let ids: Vec = producers.iter().map(|p| format!("C{}", p.0)).collect(); - format!("{{{}}}", ids.join(", ")) - }) - .collect(); - vec![ - format!("C{id}:"), - format!("O{}", candidate.operator_node.0), - format!("{:?}", candidate.timing), - format!("{:?}", candidate.materialization), - // It may produce a query's output. - if candidate.may_produce_query_output { - "output".to_owned() - } else { - String::new() - }, - reads(&producers), - ] - }) - .collect(); - write_rows(out, " ", rows); - - let output_groups: Vec = physical - .query_roots - .iter() - .map(|root| match root { - QueryRoot::Operator(group) => format!("G{}", group.0), - scalar => format!("{scalar:?}"), - }) - .collect(); - let _ = writeln!(out, "query_roots: [{}]", output_groups.join(", ")); - let _ = writeln!( - out, - "reader_constraints ({}):", - physical.reader_constraints.len() - ); - for constraint in &physical.reader_constraints { - let _ = writeln!(out, " {constraint:?}"); - } - let _ = writeln!( - out, - "pruned_candidates ({}):", - physical.pruned_candidates.len() - ); - let rows = physical - .pruned_candidates - .iter() - .map(|pruned| { - let key = &pruned.candidate; - vec![ - format!("G{}", pruned.group.0), - format!("O{}", key.operator_node.0), - format!("{:?}", key.timing), - format!("{:?}", key.materialization), - format!("{:?}", pruned.reason), - ] - }) - .collect(); - write_rows(out, " ", rows); + let _ = (out, physical); + todo!("#1207 PR 9: print each physical candidate's properties and its edges with α") } /// Stage 3's selection: the search's outcome, the selected DAG (each node's @@ -428,193 +229,12 @@ fn write_physical(out: &mut String, physical: &PhysicalASAPMemoCandidates) { /// node's charge, and each query's accepted accuracy; or why none was /// selected. fn write_selection(out: &mut String, stages: &PlanningStagesRun) { - let run = &stages.run; - let selection = match &run.selection { - Ok(selection) => selection, - Err(error) => { - let _ = writeln!(out, "error: {error}"); - return; - } - }; - let report = &selection.search_report; - let _ = writeln!( - out, - "search: {:?}, {} MILP variables, {} constraints, {} ms solving", - report.status, report.milp_variables, report.milp_constraints, report.solver_ms - ); - let dag = &selection.selected_dag; - let rows = dag - .candidate_of_node - .iter() - .enumerate() - .map(|(node, candidate)| { - let mut inputs: Vec<_> = dag - .edges - .iter() - .filter(|edge| edge.consumer.0 == node) - .collect(); - inputs.sort_by_key(|edge| edge.input); - let producers: Vec = inputs - .iter() - .map(|edge| format!("node{}", edge.producer.0)) - .collect(); - // The operator the candidate runs. - let operator = - run.physical_asap_candidates - .as_ref() - .map_or(String::new(), |physical| { - let node = physical.candidates[candidate.0].operator_node; - root_label( - &physical.operator_nodes_of_candidates_for_memo_groups[node.0] - .physical_node, - ) - }); - vec![ - format!("node{node}:"), - format!("C{}", candidate.0), - operator, - reads(&producers), - ] - }) - .collect(); - write_rows(out, "", rows); - let outputs: Vec = dag - .node_of_output_reference - .iter() - .map(|node| format!("node{}", node.0)) - .collect(); - let _ = writeln!(out, "node_of_output_reference: [{}]", outputs.join(", ")); - let cost = &selection.cost; - let _ = writeln!( - out, - "cost_per_second: {}", - cost.cost_per_second - .map_or("none".to_owned(), |cost| format!("{cost:.6}")) - ); - // Keyed by the selected plan's node ids, as Stage 2 built it. - let rows = cost - .per_node - .iter() - .map(|(node, charge)| { - vec![ - format!("#{node}"), - format!("{:?}", charge.operator), - format!("{:?}", charge.timing), - charge - .cost_per_second - .map_or("none".to_owned(), |cost| format!("{cost:.6}")), - charge.detail.clone(), - ] - }) - .collect(); - write_rows(out, " ", rows); - let _ = writeln!(out, "accepted accuracy:"); - for (query, accepted) in selection.accepted_accuracy_per_query.iter().enumerate() { - let _ = writeln!(out, " q{}: {accepted}", query + 1); - } -} - -/// The label of `node`'s own operator, without its children. -fn root_label(node: &Rc) -> String { - let (flat, _) = flatten(&[QueryRoot::Operator(node.clone())]); - flat.nodes.last().map(node_label).unwrap_or_default() -} - -/// A candidate's own nodes, from `root` down to its `input_groups`' nodes -/// (excluded), root first, joined by " ← ". -fn chain_label(memo: &Memo, root: &Rc, input_groups: &[MemoGroupId]) -> String { - let (flat, originals) = flatten(&[QueryRoot::Operator(root.clone())]); - let is_input = |id: usize| { - input_groups - .iter() - .any(|group| Rc::ptr_eq(&memo.groups[group.0 as usize].logical_node, &originals[id])) - }; - let mut labels = Vec::new(); - let mut pending = vec![flat.nodes.len() - 1]; - let mut seen = HashSet::new(); - while let Some(id) = pending.pop() { - if is_input(id) || !seen.insert(id) { - continue; - } - labels.push(node_label(&flat.nodes[id])); - pending.extend( - flat.nodes[id] - .operator - .children() - .into_iter() - .rev() - .copied(), - ); - } - labels.join(" ← ") + let _ = (out, stages); + todo!("#1207 PR 9: print the selected DAG over the new physical candidates") } -#[cfg(test)] -mod tests { - use super::*; - use crate::planner_layering_examples::example1; - use crate::planning_stages::{lower_promql_roots, run_stages_until}; - use asap_planner::PlanningModels; - use asap_types::workload::Query; - - /// One repeating `sum(rate(...[1m]))` query: every section is printed, - /// Pass 2 adds tumbling-window candidates with their layout, Stage 2 offers - /// the tumbling-window chain at ingestion time, and Stage 3 names the selected choice. - #[test] - fn prints_every_stage_of_a_repeating_rate_query() { - let mut workload = example1(); - let queries = workload.query_workload.repeating_queries.as_mut().unwrap(); - queries.truncate(1); - queries[0].query = Query("sum by (job) (rate(http_requests_total[1m]))".into()); - let roots = lower_promql_roots(&workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let stages = run_stages_until( - roots, - &entries, - &workload.data_workload.clone().unwrap(), - PlanningModels::builtin(), - RunUntil::Stage3, - ) - .unwrap(); - - let text = planning_stages_text(&stages); - - for heading in [ - "Stage 0 output", - "Stage 0 → Stage 1: DagOfLogicalMemoGroups", - "Stage 1 Pass 1 output: PerComputationLogicalMemoCandidates", - "Stage 1 Pass 2, after local window composition", - "Stage 2 input: LogicalASAPMemoCandidates", - "Stage 3 input: PhysicalASAPMemoCandidates", - "Stage 3 output: Selection", - ] { - assert!(text.contains(heading), "missing {heading}:\n{text}"); - } - // A group line names its operator and the groups it reads. - assert!(text.contains("G1: TimeRange 60s <- G0\n"), "{text}"); - // The line containing every one of `parts`, columns aside. - let has_line = |parts: &[&str]| { - text.lines() - .any(|line| parts.iter().all(|part| line.contains(part))) - }; - // Pass 2 adds a tumbling-window candidate reading the scan, its layout in its chain. - assert!( - has_line(&[ - "TumblingWindows new FinalizeExactAccumulator ← SummaryMerge ← SummaryTumblingWindowChain tumbling_window_count=6 tumbling_window_width_ms=10000", - "inputs=[G0]", - ]), - "{text}" - ); - // Stage 2 offers the tumbling-window chain at ingestion time, read from the - // ingestion-time scan. - assert!( - has_line(&["IngestionTime MatAtIngestion(Memory)", "<- {C1}"]), - "{text}" - ); - assert!( - text.contains("selected choice: G0:c0 G1:c0 G2:c0 G3:c0\n"), - "{text}" - ); - assert!(text.contains("accepted accuracy:\n q1: "), "{text}"); - } +/// The label of `group`'s own operator, without its input groups. +fn group_label(group: &MemoGroup) -> String { + let _ = group; + todo!("#1207 PR 9: label a group by its expression") } diff --git a/crates/devtools/src/rqe_atomic_costs.rs b/crates/devtools/src/rqe_atomic_costs.rs index 05681f699..ac137e19d 100644 --- a/crates/devtools/src/rqe_atomic_costs.rs +++ b/crates/devtools/src/rqe_atomic_costs.rs @@ -25,7 +25,7 @@ //! [`CostInputs::measured_operation_costs`]: asap_cost::CostInputs::measured_operation_costs use asap_cost::{MeasuredOperationCost, SummaryOperation}; -use asap_types::ir::schema::SketchParams; +use asap_types::summary_type::SketchParams; use serde_json::Value; /// A sketch-bench row that gave no entries, and why. @@ -135,10 +135,7 @@ fn sketch_params(sketch_name: &str, params: &Value) -> Result Result 0.0)); @@ -764,7 +762,7 @@ fn a_quantile_target_without_a_metric_accepts_rank_or_relative() { }]}, // Relative DDS interpolation requires this explicit domain proof. "data_workload": {"arrival": "unknown", "sources": {"m": { - "sample_interval": 1000, "series_count": 1, "values_non_negative": true + "sample_interval": 1000, "series_count": 1, "column_signs": {"value": "non_negative"} }}} }); document(&workload, &["--run-until", "stage3"]) @@ -921,14 +919,14 @@ fn deployment_capabilities_flag_sets_the_capabilities_and_shows_them() { }) ); let kll_kept = with_capabilities( - r#"{"supported_summaries": [{"family": "Kll", "readouts": ["Quantile"]}], + r#"{"supported_summaries": [{"summary_type": "Kll", "readouts": ["Quantile"]}], "query_time_results_retained": true}"#, &["--ignore-accuracy-guarantees"], ); assert_eq!( kll_kept["deployment_capabilities"], json!({ - "supported_summaries": [{ "family": "Kll", "readouts": ["Quantile"] }], + "supported_summaries": [{ "summary_type": "Kll", "readouts": ["Quantile"] }], "raw_data_retained": true, "query_time_results_retained": true, }) @@ -1026,7 +1024,7 @@ fn milp_lower_bound_is_consistent_with_the_selected_cost() { "sources": {"data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5, }}, }, @@ -1058,134 +1056,6 @@ fn milp_lower_bound_is_consistent_with_the_selected_cost() { } } -/// A dashboard panel (`max_over_time(m[5m])` every minute over 1M series -/// sampled every 1.5 s) answered as the last hour at 1-minute steps -/// (`range_evaluation`) is priced as 61 evaluations per request: the -/// selected plan costs no more than the tumbling windows kept across evaluations, and -/// the document reports their retention, 1 h + 5 min. The workload section -/// shows the range. An invalid range is refused. -#[test] -fn workload_file_with_a_range_evaluation_is_planned() { - let workload = json!({ - "query_workload": { - "language": "promql", - "query_batch": null, - "repeating_queries": [{ - "query": "max_over_time(m[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}, - "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}, - }], - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 666666.67}}, - "sources": {"m": {"series_count": 1000000, "sample_interval": 1500}}, - }, - }); - let capabilities = TempFile::new(r#"{"query_time_results_retained": true}"#); - let document = document( - &workload, - &[ - "--run-until", - "stage3", - "--ignore-accuracy-guarantees", - "--deployment-capabilities", - capabilities.path(), - ], - ); - assert_eq!( - document["workload"]["queries"][0]["requirements"]["range_evaluation"], - json!({"step_ms": 60000, "span_ms": 3600000}) - ); - let selection = &document["stage3_plan_selection"]; - let cost = selection["cost"]["cost_per_second"].as_f64().unwrap(); - // The reference: the window merged from tumbling windows kept across evaluations, - // priced under the same models. Its absolute cost is pinned in - // integration-tests/tests/range_evaluation.rs. - let kept_tumbling_windows = { - use asap_accuracy::propagation::recompose_guarantees; - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; - use asap_types::ir::ASAPOp; - use asap_types::physical::{Materialization, Medium}; - let parsed: PlanningWorkload = serde_json::from_value(workload.clone()).unwrap(); - let entries: Vec<_> = parsed.query_workload.entries().collect(); - let data = parsed.data_workload.clone().unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &lower_promql_roots(&parsed).unwrap(), - ), - ) - .unwrap(); - let exact: Vec<_> = compose_logical_candidate(&inventory, &[0]).unwrap(); - let grids: Vec<_> = entries.iter().map(|e| e.evaluation_grid_ms()).collect(); - let tumbling_windows = tumbling_window_candidates(&exact, &grids); - let windowed = window_composition_pass2_candidates(&exact, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - let deployment = asap_planner::DeploymentCapabilities { - query_time_results_retained: true, - ..Default::default() - }; - let kept = asap_physical_asap_optimization::optimize_physical_asap( - "kept-tumbling-windows", - &recompose_guarantees(&windowed).unwrap(), - &entries, - &data, - deployment.query_time_results_retained, - ) - .unwrap() - .into_iter() - .find(|plan| { - plan.dag.nodes.iter().any(|n| { - matches!( - n.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && n.materialization - == Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory) - }) - }) - .expect("a plan keeps its tumbling windows across evaluations"); - let settings = asap_cost::CostInputs::default(); - let priced = asap_devtools::memo_pricing::reference_select( - std::slice::from_ref(&kept), - &entries, - &data, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(&deployment), - ) - .unwrap(); - priced.costs[&priced.selected].cost_per_second.unwrap() - }; - assert!( - cost <= kept_tumbling_windows * (1.0 + 1e-9), - "selected {SELECTED} costs {cost}, more than the kept tumbling windows' {kept_tumbling_windows}" - ); - assert!(selection["tumbling_window_retention_ms"] - .as_object() - .unwrap() - .values() - .any(|retention| retention.as_u64() == Some(3_900_000))); - - let mut invalid = workload.clone(); - invalid["query_workload"]["repeating_queries"][0]["range_evaluation"]["step_ms"] = json!(0); - let invalid = TempFile::json(&invalid); - let stderr = refused(&["--workload", invalid.path(), "--run-until", "stage0"]); - assert!( - stderr.contains("step_ms must be greater than zero"), - "{stderr}" - ); -} - /// The default is to enforce accuracy guarantees: Pass 1 prunes every /// sketch candidate of an average of p99s, which has no guarantee /// (`NoErrorBoundForQueryTarget`). With `--ignore-accuracy-guarantees`, the @@ -1260,7 +1130,7 @@ fn workload_set_index_and_files_validate() { ); } if let Some(capabilities) = set.get("deployment_capabilities") { - serde_json::from_value::( + serde_json::from_value::( capabilities.clone(), ) .unwrap(); @@ -1428,7 +1298,7 @@ fn undeclared_supported_summaries_default_to_the_executors_families() { /// Two windowed quantiles over one metric (Q1 over 1 m, Q2 over 5 m, both /// every 10 s): with a shared scan, Stage 2 also offers the scan built once /// per evaluation, which reads the 5 m window once (≈0.1364/s) instead of -/// once per reader (0.2203/s); the selected plan costs at most 0.143/s. +/// once per query (0.2203/s); the selected plan costs at most 0.143/s. #[test] fn a_scan_two_queries_share_is_offered_built_once_per_evaluation() { let workload = json!({ @@ -1459,230 +1329,6 @@ fn a_scan_two_queries_share_is_offered_built_once_per_evaluation() { assert!(cost <= 0.143, "{cost}"); } -/// A scan and a time range that already have every label of their metric -/// are not widened, so no memo group's candidate reads its own group, and -/// Stage 2's candidates, whose producers are its input groups' candidates, -/// read no candidate that reads them back. -#[test] -fn widened_scans_keep_memo_groups_and_stage2_candidates_acyclic() { - use asap_logical_asap_optimization::DeploymentCapabilities; - let workload: PlanningWorkload = serde_json::from_value(json!({ - "query_workload": { - "language": "promql", - "query_batch": null, - "repeating_queries": [ - every_10s_within_1_percent("sum by (job, region) (rate(data[5m]))"), - every_10s_within_1_percent("sum by (job) (rate(data[5m]))"), - every_10s_within_1_percent("sum by (job, region) (data)"), - every_10s_within_1_percent("sum by (job) (data)"), - ], - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "counter", "sample_interval": 15000}}, - }, - })) - .unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let deployment = DeploymentCapabilities { - raw_data_retained: true, - ..DeploymentCapabilities::default() - }; - let run = run_stages_until( - lower_promql_roots(&workload).unwrap(), - &entries, - workload.data_workload.as_ref().unwrap(), - asap_planner::PlanningModels::builtin().with_deployment_capabilities(&deployment), - RunUntil::Stage2, - ) - .unwrap() - .run; - let memo = &run.logical_asap_candidates.as_ref().unwrap().memo; - assert!(memo - .candidates - .iter() - .flatten() - .any(|candidate| candidate.origin - == asap_types::stages::OriginOfCandidateForMemoGroup::WidenedScans)); - // Whether `to` is reachable from `from` along `edges`. - fn reaches(edges: &dyn Fn(usize) -> Vec, from: usize, to: usize) -> bool { - let mut seen = HashSet::new(); - let mut pending = edges(from); - while let Some(next) = pending.pop() { - if next == to { - return true; - } - if seen.insert(next) { - pending.extend(edges(next)); - } - } - false - } - let group_reads = |group: usize| -> Vec { - memo.candidates[group] - .iter() - .flat_map(|candidate| candidate.input_groups.iter().map(|g| g.0 as usize)) - .collect() - }; - for group in 0..memo.groups.len() { - assert!( - !reaches(&group_reads, group, group), - "memo group {group} reads itself" - ); - } - let physical = run.physical_asap_candidates.as_ref().unwrap(); - let candidate_reads = |candidate: usize| -> Vec { - physical.candidates[candidate] - .producer_candidates_per_input - .iter() - .flat_map(|producers| producers.iter().map(|id| id.0)) - .collect() - }; - for candidate in 0..physical.candidates.len() { - assert!( - !reaches(&candidate_reads, candidate, candidate), - "Stage 2 candidate {candidate} reads itself" - ); - } -} - -/// #840's workload: Q1 and Q2 window `data` (1 m and 5 m), Q3 and Q4 read -/// its latest samples by (host, job) and by job, so the four scans project -/// different labels. The widening rule's one scan containing every label -/// installs, compiles and is charged once per evaluation, and the selected -/// plan costs no more than it or than the partial-sharing plan (#994). -#[test] -fn scans_projecting_different_labels_are_shared() { - let workload = json!({ - "query_workload": { - "language": "promql", - "query_batch": null, - "repeating_queries": [ - every_10s_within_1_percent("quantile_over_time(0.5, data[1m])"), - every_10s_within_1_percent("quantile_over_time(0.9, data[5m])"), - every_10s_within_1_percent("quantile by (host, job) (0.5, data)"), - every_10s_within_1_percent("quantile by (job) (0.5, data)"), - ], - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "gauge", "sample_interval": 15000}}, - }, - }); - let document = document( - &workload, - &["--run-until", "stage3", "--ignore-accuracy-guarantees"], - ); - let workload: PlanningWorkload = serde_json::from_value(workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.as_ref().unwrap(); - let roots = lower_promql_roots(&workload).unwrap(); - use asap_logical_asap_optimization::test_support::with_union_scan_labels; - use asap_types::physical::{Materialization, Medium}; - // Plans `roots` with identical sub-DAGs made one node, as the memo's - // groups are, and a node read by several roots kept for the evaluation. - let plan = |name: &str, roots: &[QueryRoot]| { - let shared: Vec<_> = asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, std::rc::Rc::clone(node)), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect(); - let physical = asap_physical_asap_optimization::optimize_physical_asap_assigned( - name, - &shared, - &entries, - data, - false, - &mut |readers| { - Some(if readers.len() > 1 { - Materialization::MatAtQuery(Medium::Memory) - } else { - Materialization::NotMaterialized - }) - }, - ) - .unwrap() - .expect("a node read by several roots can be kept for the evaluation"); - let offered = asap_devtools::memo_pricing::reference_select( - std::slice::from_ref(&physical), - &entries, - data, - asap_planner::PlanningModels::builtin(), - ) - .unwrap(); - (physical, offered) - }; - // The widened scan: one scan with every label, shared by all four. - let widened = with_union_scan_labels(&roots).unwrap().unwrap(); - let (physical, offered) = plan("widened-scan-rule", &widened); - // The partial-sharing reference: the two windows share a narrow scan, and - // the two instant selectors a scan widened to their labels. - let mut partial = roots[..2].to_vec(); - partial.extend(with_union_scan_labels(&roots[2..]).unwrap().unwrap()); - let (_, reference) = plan("partial-sharing", &partial); - let (installed, _) = asap_planner::installable_plan_of( - &physical.roots, - None, - &offered.tumbling_window_retention_ms, - &entries, - ) - .unwrap(); - installed.validate().unwrap(); - installed.check_compiles().unwrap(); - let dag = &physical.dag; - let scans: Vec<_> = dag - .nodes - .iter() - .filter(|n| n.payload.kind_name() == "Scan") - .collect(); - let [scan] = &scans[..] else { - panic!("one scan: {scans:?}") - }; - let field_names: Vec<&str> = scan - .output_schema - .fields - .iter() - .map(|f| f.name.as_str()) - .collect(); - assert_eq!( - field_names, - ["ts", "value", "host", "job", "$promql_series_identity"] - ); - let cost = &offered.costs[&offered.selected]; - let scan_cost = &cost.per_node[&scan.id]; - assert_eq!(scan_cost.multiplier, 1, "{scan_cost:?}"); - // 10,000 series × 20 samples in 5 m, once per 10 s evaluation. - let detail = &scan_cost.detail; - assert!( - detail.starts_with("scan 200000 samples; x 0.1000/s"), - "{detail}" - ); - let shared_cost = cost.cost_per_second.unwrap(); - assert!(shared_cost.is_finite() && shared_cost > 0.0); - let selected_cost = document["stage3_plan_selection"]["cost"]["cost_per_second"] - .as_f64() - .unwrap(); - let reference_cost = reference.costs[&reference.selected] - .cost_per_second - .unwrap(); - assert!( - selected_cost <= shared_cost, - "{selected_cost} > {shared_cost}" - ); - assert!( - selected_cost <= reference_cost * (1.0 + 1e-9), - "{selected_cost} > partial sharing {reference_cost}" - ); -} - /// The search is the candidate-connection search; its report has no /// legacy route. #[test] diff --git a/crates/devtools/tests/asap_planner_acceptance.rs b/crates/devtools/tests/asap_planner_acceptance.rs index 7043d0767..7567cbde3 100644 --- a/crates/devtools/tests/asap_planner_acceptance.rs +++ b/crates/devtools/tests/asap_planner_acceptance.rs @@ -73,13 +73,10 @@ const SECTIONS: [(&str, &[&str]); 5] = [ "stage0", &[ "Stage 0 output: the frontends' roots", - "Stage 0 → Stage 1: DagOfLogicalMemoGroups", + "Stage 0 → Stage 1: Memo (the roots hash-consed, without candidates)", ], ), - ( - "stage1_pass1", - &["Stage 1 Pass 1 output: PerComputationLogicalMemoCandidates"], - ), + ("stage1_pass1", &["Stage 1 Pass 1 output: Memo"]), ( "stage1_pass2", &[ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json index d803d90f7..5cb411f72 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json @@ -721,7 +721,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -861,8 +861,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -954,7 +954,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1044,7 +1044,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1112,7 +1112,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1197,7 +1197,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1257,7 +1257,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json index b986b993f..4d42935a1 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json @@ -880,7 +880,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1034,8 +1034,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1127,7 +1127,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1217,7 +1217,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1285,7 +1285,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1356,7 +1356,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1404,7 +1404,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1528,8 +1528,8 @@ { "guarantee": null, "id": 7, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1613,7 +1613,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1695,7 +1695,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1755,7 +1755,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1826,7 +1826,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1874,7 +1874,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1932,7 +1932,7 @@ } }, "id": 13, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1971,7 +1971,7 @@ { "guarantee": null, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2040,7 +2040,7 @@ } }, "id": 15, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json index 99affb00a..8cca94ae9 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json @@ -569,7 +569,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -709,8 +709,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -802,7 +802,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -892,7 +892,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -960,7 +960,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1045,7 +1045,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1105,7 +1105,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ @@ -1172,7 +1172,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ @@ -1211,7 +1211,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ @@ -1280,7 +1280,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json index 16f3c5c73..33b444675 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json @@ -328,7 +328,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -438,7 +438,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -495,7 +495,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -566,7 +566,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -614,7 +614,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -653,7 +653,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -722,7 +722,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json index 2a4d99a83..8f1b22d92 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json @@ -133,7 +133,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -274,7 +274,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json index cc49899a7..a491b2f63 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json @@ -7148,7 +7148,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7319,7 +7319,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7391,8 +7391,8 @@ { "guarantee": null, "id": 2, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7484,7 +7484,7 @@ { "guarantee": null, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7574,7 +7574,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7642,7 +7642,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7713,7 +7713,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7761,7 +7761,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7902,7 +7902,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7966,8 +7966,8 @@ { "guarantee": null, "id": 9, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -8051,7 +8051,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8133,7 +8133,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8193,7 +8193,7 @@ { "guarantee": null, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8264,7 +8264,7 @@ } }, "id": 13, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8312,7 +8312,7 @@ } }, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json index 2d5150c66..2e30b8d58 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json @@ -7012,7 +7012,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7166,8 +7166,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7259,7 +7259,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7349,7 +7349,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7417,7 +7417,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7488,7 +7488,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7536,7 +7536,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7660,8 +7660,8 @@ { "guarantee": null, "id": 7, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7745,7 +7745,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7827,7 +7827,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7887,7 +7887,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7958,7 +7958,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8006,7 +8006,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json index 6b44c713b..3adc6ca62 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json @@ -219,7 +219,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -329,7 +329,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -386,7 +386,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -457,7 +457,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json index 8546dd2b2..3ffc88b31 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json @@ -694,7 +694,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -835,7 +835,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -899,8 +899,8 @@ { "guarantee": null, "id": 2, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -984,7 +984,7 @@ { "guarantee": null, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1066,7 +1066,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1126,7 +1126,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1197,7 +1197,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json index b7c91f320..e0a83a8d0 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json @@ -630,7 +630,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -754,8 +754,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -839,7 +839,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -921,7 +921,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -981,7 +981,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1052,7 +1052,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json index 2ad030eed..5156b5315 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json @@ -560,7 +560,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -684,8 +684,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -769,7 +769,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -851,7 +851,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -911,7 +911,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -982,7 +982,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json index 9f4988b12..c69ea0939 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json @@ -5332,7 +5332,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5486,8 +5486,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -5579,7 +5579,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5669,7 +5669,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5737,7 +5737,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5808,7 +5808,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5856,7 +5856,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5980,8 +5980,8 @@ { "guarantee": null, "id": 7, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -6065,7 +6065,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6147,7 +6147,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6207,7 +6207,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6278,7 +6278,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6326,7 +6326,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json index e33bf4553..fbdeba67f 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json @@ -2618,7 +2618,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2711,8 +2711,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2788,7 +2788,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2862,7 +2862,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2914,7 +2914,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2985,7 +2985,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json index e119b2b13..e87f425a7 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json @@ -219,7 +219,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -329,7 +329,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -386,7 +386,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -457,7 +457,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/cross_language.rs b/crates/devtools/tests/cross_language.rs index 3309dda27..4d28c0b8d 100644 --- a/crates/devtools/tests/cross_language.rs +++ b/crates/devtools/tests/cross_language.rs @@ -15,7 +15,6 @@ use asap_devtools::{lower_promql_with_sample_interval, lower_sql, SqlCatalog}; use asap_types::ir::operator::{AggIntent, GroupKeys}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; use std::rc::Rc; @@ -42,13 +41,13 @@ fn catalog() -> SqlCatalog { } async fn sql(q: &str) -> Rc { - lower_sql(q, &catalog(), AccuracyTarget::exact()) + lower_sql(q, &catalog()) .await .unwrap_or_else(|e| panic!("SQL {q:?} failed to lower: {e:?}")) } fn promql(q: &str) -> Rc { - lower_promql_with_sample_interval(q, AccuracyTarget::exact(), 1_000) + lower_promql_with_sample_interval(q, 1_000) .unwrap_or_else(|e| panic!("PromQL {q:?} failed to lower: {e:?}")) } @@ -75,8 +74,7 @@ fn heavy_hitter(qe: &OperatorNode) -> Option<(usize, GroupKeys)> { else { return None; }; - matches!(inner.as_slice(), [AggIntent::Count { .. }]) - .then(|| (*k, reduction.expect_reduce().clone())) + matches!(inner.as_slice(), [AggIntent::Count]).then(|| (*k, reduction.expect_reduce().clone())) } #[tokio::test] diff --git a/crates/devtools/tests/main.rs b/crates/devtools/tests/main.rs index 4952ae75e..ed12546d0 100644 --- a/crates/devtools/tests/main.rs +++ b/crates/devtools/tests/main.rs @@ -5,6 +5,5 @@ mod asap_planner; mod asap_planner_acceptance; mod cross_language; -mod memo_pricing; mod planner_fuzz; mod viewer_contract; diff --git a/crates/devtools/tests/memo_pricing.rs b/crates/devtools/tests/memo_pricing.rs deleted file mode 100644 index 1bdc9e850..000000000 --- a/crates/devtools/tests/memo_pricing.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! Pricing a memo's enumerated choices: on small workloads every choice is -//! priced, and none that is valid is cheaper than the plan the search -//! selected. -use asap_cost::CostInputs; -use asap_devtools::memo_pricing::{price_choices, PricedChoice, SelectedBaseline}; -use asap_devtools::planner_layering_examples::example1; -use asap_devtools::planning_stages::{lower_promql_roots, run_stages_until, RunUntil}; -use asap_logical_asap_optimization::enumerate; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; -use asap_logical_asap_optimization::test_support::try_populate_pass1; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_planner::PlanningModels; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::QueryRoot; -use asap_types::stages::Memo; -use asap_types::workload::{ - DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, QueryWorkloadEntry, RepeatedDemand, RepeatingEntry, - RepetitionInterval, -}; - -/// The issue's workload file: the query every minute, ε = 0.01, δ = 0.001. -const RATIO_OF_QUANTILES_WORKLOAD: &str = - "../../tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json"; - -const RATIO_OF_QUANTILES: &str = "avg by (job) (quantile_over_time(0.5, data[5m]) \ - / quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m]))"; - -/// One query evaluated every minute at max_error 0.01, failure probability -/// 0.001, over 1,000 series of `data`, each sampled every 15 s. -fn ratio_of_quantiles() -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query(RATIO_OF_QUANTILES.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(60_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget::max_error(0.01).failure_probability(0.001), - max_latency_ms: None, - }, - predictability: Predictability::Unknown, - range_evaluation: None, - }]), - }, - data_workload: Some({ - let mut data = DataWorkload::sampled_every(["data".to_string()], DurationMs(15_000)); - for facts in data.sources.values_mut() { - facts.series_count = Some(1_000); - } - data - }), - } -} - -/// The workload's roots, entries and data, and its Pass 1 memo. -struct Planned { - roots: Vec, - entries: Vec, - data: DataWorkload, - memo: Memo, -} - -/// `asap_planner` lowers each series' identity as a column -/// (`lower_promql_roots`); without it (`series_identity` false, as the -/// memo-enumeration tests lower), the ratio's two `quantile 0.5` have no -/// provable unique key, but they are still one group. -fn planned(workload: &PlanningWorkload, series_identity: bool) -> Planned { - let roots = if series_identity { - lower_promql_roots(workload).unwrap() - } else { - asap_frontend_promql::lower_promql_query_workload(workload) - .unwrap() - .into_iter() - .collect() - }; - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - Planned { - roots, - entries: workload.query_workload.entries().collect(), - data: workload.data_workload.clone().unwrap(), - memo: try_populate_pass1(initial, &inventory).unwrap(), - } -} - -/// `asap_planner`'s models: the executor's summary families, -/// `settings` (accuracy guarantees ignored unless enforced). -fn models<'a>( - settings: &'a Settings, - deployment: &'a DeploymentCapabilities, -) -> PlanningModels<'a> { - PlanningModels::builtin() - .with_cost_inputs(&settings.cost_inputs) - .with_deployment_capabilities(deployment) - .with_ignore_accuracy_guarantees(settings.ignore_accuracy_guarantees) -} - -fn deployment() -> DeploymentCapabilities { - DeploymentCapabilities { - supported_summaries: Some(asap_executor::supported_summary_families()), - ..DeploymentCapabilities::default() - } -} - -/// `asap_planner`'s cost inputs, and whether it ignores accuracy -/// guarantees. -struct Settings { - cost_inputs: CostInputs, - ignore_accuracy_guarantees: bool, -} - -fn settings(enforce_accuracy_guarantees: bool) -> Settings { - Settings { - cost_inputs: CostInputs::default(), - ignore_accuracy_guarantees: !enforce_accuracy_guarantees, - } -} - -/// `asap_planner`'s selection of `planned`'s workload. -fn selected(planned: &Planned, models: PlanningModels<'_>) -> SelectedBaseline { - let run = run_stages_until( - planned.roots.clone(), - &planned.entries, - &planned.data, - models, - RunUntil::Stage3, - ) - .unwrap(); - SelectedBaseline::of(run.run.selection.as_ref().expect("a selection")) -} - -/// Every choice of `planned`'s Pass 1 memo (the enumeration is complete), -/// priced against `baseline`. -fn price_every_choice( - planned: &Planned, - models: PlanningModels<'_>, - baseline: SelectedBaseline, -) -> Vec { - let enumeration = enumerate(&planned.memo, 100_000); - assert!(enumeration.complete); - price_choices( - &planned.memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models, - Some(baseline), - ) -} - -/// Every choice of `workload`'s Pass 1 memo, as `asap_planner` builds it -/// (the enumeration is complete), priced against the plan `asap_planner` -/// selects: none is cheaper, every valid one costs at least as much, and -/// the others say why they were rejected. Returns the priced choices. -fn assert_no_pass1_choice_is_cheaper( - workload: &PlanningWorkload, - enforce_accuracy_guarantees: bool, -) -> Vec { - let planned = planned(workload, true); - let (settings, deployment) = (settings(enforce_accuracy_guarantees), deployment()); - let models = models(&settings, &deployment); - let run = run_stages_until( - planned.roots.clone(), - &planned.entries, - &planned.data, - models, - RunUntil::Stage3, - ) - .unwrap(); - let selected = SelectedBaseline::of(run.run.selection.as_ref().expect("a selection")); - let memo = &run.run.per_computation_candidates.as_ref().unwrap().memo; - let enumeration = enumerate(memo, 100_000); - assert!(enumeration.complete); - let priced = price_choices( - memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models, - Some(selected), - ); - let selected = selected.cost_per_second.unwrap(); - for choice in &priced { - assert!(!choice.cheaper_than_selected, "{choice:?}"); - if choice.valid { - assert!(choice.cost_per_second.unwrap() >= selected, "{choice:?}"); - assert_eq!(choice.rejection, None); - } else { - assert!(choice.rejection.is_some(), "{choice:?}"); - } - } - assert!(priced.iter().any(|choice| choice.valid)); - priced -} - -/// #509 Example 1: its Pass 1 memo stands for 3 DAGs -/// (Q2's top-k group: exact, Count-Min with a heap and Count Sketch with a -/// heap); -/// Q2's summary reads miss its 100 ms latency bound, and no valid DAG is -/// cheaper than the selection. -#[test] -fn example1_no_pass1_choice_is_cheaper_than_the_selected_plan() { - let choices = assert_no_pass1_choice_is_cheaper(&example1(), false); - assert_eq!(choices.len(), 3); - assert!(choices - .iter() - .any(|c| c.rejection.as_deref() == Some("latency_bound_exceeded"))); -} - -/// The issue's ratio of quantiles as `asap_planner` lowers it (series -/// identity: the two `quantile 0.5` share one group): no Pass 1 DAG is -/// cheaper than the selection, with accuracy guarantees ignored or -/// enforced (the default). Enforced, Pass 1 prunes every summary: no query -/// has a guarantee through `/` and `+` over a KLL rank error or a DDSketch -/// relative one, so every listed DAG is exact and valid. -#[test] -fn ratio_of_quantiles_no_pass1_choice_is_cheaper_than_the_selected_plan() { - let workload: PlanningWorkload = - serde_json::from_str(&std::fs::read_to_string(RATIO_OF_QUANTILES_WORKLOAD).unwrap()) - .unwrap(); - let ignored = assert_no_pass1_choice_is_cheaper(&workload, false); - assert!(ignored.len() >= 25); - let enforced = assert_no_pass1_choice_is_cheaper(&workload, true); - assert!(!enforced.is_empty()); - assert!(enforced.iter().all(|choice| choice.valid)); -} - -/// Lowered without series identity, the two `quantile 0.5` have no unique -/// key but return the same rows, so they are still one group: 5 × 5 = 25 -/// DAGs (two distinct approximate inputs), 9 of them sized for the whole -/// budget. The executor compiles none of their per-series summaries without -/// the series identity, so only the exact DAG compiles and is valid. Its price is not -/// compared with the plan selected for the series-identity lowering: its -/// rows carry no identity column, so it prices lower than the same exact -/// DAG with the identity (#1035). -#[test] -fn ratio_of_quantiles_25_pass1_choices_without_identity_only_exact_is_valid() { - let without_identity = planned(&ratio_of_quantiles(), false); - let enumeration = enumerate(&without_identity.memo, 100_000); - let whole_budget = enumeration - .all_chosen_candidates - .iter() - .filter(|choice| { - choice.iter().all(|(group, &index)| { - without_identity.memo.candidates[group.0 as usize][index].accuracy_budget_parts == 1 - }) - }) - .count(); - assert_eq!( - (enumeration.all_chosen_candidates.len(), whole_budget), - (25, 9) - ); - let with_identity = planned(&ratio_of_quantiles(), true); - let deployment = deployment(); - for enforce in [false, true] { - let settings = settings(enforce); - let models = models(&settings, &deployment); - let baseline = selected(&with_identity, models); - let priced = price_every_choice(&without_identity, models, baseline); - assert_eq!(priced.iter().filter(|p| p.valid).count(), 1); - } -} - -/// A selection one step worse than the cheapest valid choice (the -/// next-cheapest cost) makes exactly the cheaper choices -/// `cheaper_than_selected`. -#[test] -fn a_worse_selection_is_caught_by_cheaper_than_selected() { - let planned = planned(&example1(), true); - let (settings, deployment) = (settings(false), deployment()); - let models = models(&settings, &deployment); - let choices = enumerate(&planned.memo, 100).all_chosen_candidates; - let price = |selected| { - price_choices( - &planned.memo, - &choices, - &planned.roots, - &planned.entries, - &planned.data, - models, - selected, - ) - }; - let unpriced = price(None); - assert!(unpriced.iter().all(|p| !p.cheaper_than_selected)); - let mut costs: Vec = unpriced.iter().filter_map(|p| p.cost_per_second).collect(); - costs.sort_by(f64::total_cmp); - costs.dedup(); - assert!(costs.len() >= 2, "two valid choices of different cost"); - let worse = SelectedBaseline { - cost_per_second: Some(costs[1]), - calibration_factor: None, - }; - let priced = price(Some(worse)); - for (before, after) in unpriced.iter().zip(&priced) { - assert_eq!(before.cost_per_second, after.cost_per_second); - assert_eq!( - after.cheaper_than_selected, - after.cost_per_second == Some(costs[0]) - ); - } - assert!(priced.iter().any(|p| p.cheaper_than_selected)); -} - -/// A query whose root is a scalar expression reading two operator DAGs: -/// each reference is replaced by its own root group's composed DAG, so -/// every choice composes and is priced. -#[test] -fn scalar_root_choices_are_composed_and_priced() { - let mut workload = ratio_of_quantiles(); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = Query( - "scalar(sum(quantile_over_time(0.5, data[5m]))) + scalar(max(rate(data[1m])))".into(), - ); - let planned = planned(&workload, true); - assert!(matches!(planned.roots[0], QueryRoot::Scalar(_))); - let (settings, deployment) = (settings(false), deployment()); - let enumeration = enumerate(&planned.memo, 100_000); - assert!(enumeration.complete && enumeration.all_chosen_candidates.len() > 1); - let priced = price_choices( - &planned.memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models(&settings, &deployment), - None, - ); - for p in &priced { - assert_ne!(p.rejection.as_deref(), Some("compose_failed"), "{p:?}"); - } - assert!(priced.iter().any(|p| p.valid)); -} - -/// A choice whose selected plan installation refuses for a reason other -/// than the executor (here a raw-data window under a subquery, which no -/// runtime can place) is `not_installable`, never valid, so it is never -/// flagged cheaper than the selected plan. -#[test] -fn a_choice_installation_refuses_is_not_installable() { - let mut workload = ratio_of_quantiles(); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query("max_over_time(rate(data[5m])[1h:1m])".into()); - workload - .query_workload - .repeating_queries - .as_mut() - .unwrap() - .truncate(1); - let planned = planned(&workload, true); - let (settings, deployment) = (settings(false), deployment()); - let enumeration = enumerate(&planned.memo, 100_000); - let priced = price_choices( - &planned.memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models(&settings, &deployment), - None, - ); - assert!(!priced.is_empty()); - for p in &priced { - assert!(!p.valid && !p.cheaper_than_selected, "{p:?}"); - } - let refused: Vec<_> = priced - .iter() - .filter(|p| p.rejection.as_deref() == Some("not_installable")) - .collect(); - assert!(!refused.is_empty(), "{priced:?}"); - for p in refused { - let reason = p.rejection_reason.as_deref().unwrap(); - assert!( - reason.contains("a subquery or a TimeShift above the range"), - "{reason}" - ); - } -} diff --git a/crates/devtools/tests/planner_fuzz.rs b/crates/devtools/tests/planner_fuzz.rs index 73df7cc00..232d75332 100644 --- a/crates/devtools/tests/planner_fuzz.rs +++ b/crates/devtools/tests/planner_fuzz.rs @@ -182,7 +182,7 @@ fn campaign_writes_summary_coverage_and_drafts() { fn generated(spec: QuerySpec, approximate: bool) -> FuzzWorkload { use asap_devtools::planner_fuzz::generate::generated_workload; use asap_devtools::planner_fuzz::{FuzzQuery, Language}; - use asap_types::ir::properties::AccuracyTarget; + use asap_types::accuracy::AccuracyTarget; let language = match spec { QuerySpec::Promql(_) => Language::Promql, QuerySpec::Sql(_) => Language::Sql, @@ -382,7 +382,7 @@ fn longer_window_rollup_witness_prices() { fn other_frequency_statistic_witness_prices_from_one_univmon() { use asap_devtools::planner_fuzz::generate::generated_workload; use asap_devtools::planner_fuzz::{FuzzQuery, Language}; - use asap_types::ir::properties::AccuracyTarget; + use asap_types::accuracy::AccuracyTarget; let l2 = QuerySpec::Sql(SqlSpec { table: "metrics".into(), aggregate: SqlAggregate::FrequencyL2("src_ip".into()), diff --git a/crates/devtools/tests/sketch_plan_snapshots.rs b/crates/devtools/tests/sketch_plan_snapshots.rs index e3c9d1be5..1ab0097d1 100644 --- a/crates/devtools/tests/sketch_plan_snapshots.rs +++ b/crates/devtools/tests/sketch_plan_snapshots.rs @@ -6,7 +6,7 @@ //! queries or nested windows, answer a ratio of two quantiles from one tumbling window //! chain, and materialize tumbling windows, under the MVP deployment (raw data not //! retained) and raw data retained. A change to sketch kinds, parameters, -//! sizes, sharing, timing, materialization or cost shows up as a snapshot +//! sizes, sharing, timing, lifetime or cost shows up as a snapshot //! diff. //! //! `tests/sketch_plan_snapshots/manifest.json` lists each workload by id: diff --git a/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json b/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json index 225a81a8a..f27b75631 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json +++ b/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json @@ -378,7 +378,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -488,8 +488,8 @@ } }, "id": 1, - "materialization": { - "MatAtQuery": "Memory" + "lifetime": { + "OneEvaluation": "Memory" }, "output_schema": { "closed": true, @@ -564,7 +564,7 @@ } }, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -634,7 +634,7 @@ { "guarantee": null, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -750,7 +750,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -834,7 +834,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json index 9b4fb2c61..54a2d6f95 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json @@ -2802,7 +2802,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2895,8 +2895,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2986,7 +2986,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3085,7 +3085,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3176,7 +3176,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3249,7 +3249,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3347,7 +3347,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3438,7 +3438,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json index 3baaa111e..37361b98d 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json @@ -1898,7 +1898,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2007,8 +2007,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2106,7 +2106,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2212,7 +2212,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2311,7 +2311,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2398,7 +2398,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2573,7 +2573,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2616,7 +2616,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2701,7 +2701,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2761,7 +2761,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2932,7 +2932,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3092,7 +3092,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json index 4f357790a..6583082f6 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json @@ -1461,7 +1461,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1554,8 +1554,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1647,7 +1647,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1739,7 +1739,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1823,7 +1823,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json index 04ea77719..d569fff3a 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json @@ -12158,7 +12158,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12267,8 +12267,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -12366,7 +12366,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12472,7 +12472,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12571,7 +12571,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12658,7 +12658,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12833,7 +12833,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12876,7 +12876,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12961,7 +12961,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -13021,7 +13021,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -13192,7 +13192,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -13352,7 +13352,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json index f9db2cbc5..95d124a1c 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json @@ -6490,7 +6490,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6599,8 +6599,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -6698,7 +6698,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6804,7 +6804,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6903,7 +6903,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6990,7 +6990,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7165,7 +7165,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7208,7 +7208,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7293,7 +7293,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7353,7 +7353,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7524,7 +7524,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7684,7 +7684,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7759,7 +7759,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7868,8 +7868,8 @@ { "guarantee": null, "id": 13, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7953,7 +7953,7 @@ { "guarantee": null, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8035,7 +8035,7 @@ } }, "id": 15, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8112,7 +8112,7 @@ } }, "id": 16, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8201,7 +8201,7 @@ } }, "id": 17, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8283,7 +8283,7 @@ } }, "id": 18, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json index 6de034914..94d046d27 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json @@ -7250,7 +7250,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7359,8 +7359,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7458,7 +7458,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7564,7 +7564,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7663,7 +7663,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7750,7 +7750,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7925,7 +7925,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7968,7 +7968,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8053,7 +8053,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8113,7 +8113,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8284,7 +8284,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8444,7 +8444,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8519,7 +8519,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8628,8 +8628,8 @@ { "guarantee": null, "id": 13, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -8713,7 +8713,7 @@ { "guarantee": null, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8795,7 +8795,7 @@ } }, "id": 15, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8872,7 +8872,7 @@ } }, "id": 16, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8961,7 +8961,7 @@ } }, "id": 17, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -9043,7 +9043,7 @@ } }, "id": 18, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json index 0d170be65..266118af6 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json @@ -2890,7 +2890,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2983,8 +2983,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -3074,7 +3074,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3172,7 +3172,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3263,7 +3263,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3336,7 +3336,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3434,7 +3434,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3525,7 +3525,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3598,7 +3598,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json index 227c9d39c..fa094b72a 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json @@ -2802,7 +2802,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2895,8 +2895,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2986,7 +2986,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3084,7 +3084,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3175,7 +3175,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3248,7 +3248,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3346,7 +3346,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3437,7 +3437,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json index b49a890c2..3d269fb2c 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json @@ -1571,7 +1571,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1680,8 +1680,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1765,7 +1765,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1847,7 +1847,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1907,7 +1907,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1984,7 +1984,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2032,7 +2032,7 @@ { "guarantee": null, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-grouped.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-grouped.json index 656dabd73..1dd0566c5 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-grouped.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-grouped.json @@ -32,7 +32,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-single.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-single.json index 65bc2ffbe..0455fd361 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-single.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-single.json @@ -32,7 +32,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-long-w01-q3.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-long-w01-q3.json index 33c2c2a3c..d0006c2da 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-long-w01-q3.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-long-w01-q3.json @@ -34,7 +34,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w01-q3-q8.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w01-q3-q8.json index 4e14096a2..1bd44805f 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w01-q3-q8.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w01-q3-q8.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w02-q2-q7.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w02-q2-q7.json index 9ade0f5be..35784b296 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w02-q2-q7.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w02-q2-q7.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/ratio-of-quantiles-1h.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/ratio-of-quantiles-1h.json index 85976d570..24f83d95c 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/ratio-of-quantiles-1h.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/ratio-of-quantiles-1h.json @@ -34,7 +34,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/topk-rate.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/topk-rate.json index 43fc749f2..33bd85a34 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/topk-rate.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/topk-rate.json @@ -32,7 +32,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/viewer_contract.rs b/crates/devtools/tests/viewer_contract.rs index 559223d2e..316f0905e 100644 --- a/crates/devtools/tests/viewer_contract.rs +++ b/crates/devtools/tests/viewer_contract.rs @@ -15,11 +15,11 @@ use asap_types::ir::operator::{ TimeShift, WindowFuncKind, }; use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, ScalarValue}; -use asap_types::ir::schema::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; use asap_types::ir::schema::{DataType, FieldDataType, Schema}; use asap_types::ir::{ ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, }; +use asap_types::summary_type::{SketchStatistic, SummaryUpdate}; /// One instance of every `NonASAPOp` variant, children elided. The /// exhaustive match fails to compile when a variant is added, until a sample @@ -171,10 +171,9 @@ fn asap_samples() -> Vec> { let samples = vec![ ASAPOp::SummaryAgg { child: (), - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, }, ASAPOp::SummaryEstimate { @@ -200,18 +199,15 @@ fn asap_samples() -> Vec> { }, ASAPOp::SummaryMerge { children: vec![], - newest_tumbling_windows: None, group_by: None, }, ASAPOp::SummaryTumblingWindowChain { child: (), - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, tumbling_window_width_ms: 60_000, - tumbling_window_count: 5, }, ASAPOp::SummarySubtract { left: (), @@ -225,7 +221,7 @@ fn asap_samples() -> Vec> { outer: (), inner: (), key: 0, - family, + summary_type: family, }, ASAPOp::Extension { child: (), @@ -345,7 +341,6 @@ fn viewer_knows_every_view_refresh_and_rejection_cause() { let causes = [ C::AccuracyTargetMissed, - C::CountMinWeightsNotProvenNonNegative, C::NoAccuracyGuarantee, C::LatencyBoundExceeded, C::LatencyOverRepeatInterval, @@ -360,7 +355,6 @@ fn viewer_knows_every_view_refresh_and_rejection_cause() { for cause in causes { match cause { C::AccuracyTargetMissed - | C::CountMinWeightsNotProvenNonNegative | C::NoAccuracyGuarantee | C::LatencyBoundExceeded | C::LatencyOverRepeatInterval diff --git a/crates/executor/src/capability.rs b/crates/executor/src/capability.rs index f7a2e3afd..61ee2d8bc 100644 --- a/crates/executor/src/capability.rs +++ b/crates/executor/src/capability.rs @@ -8,12 +8,12 @@ //! //! Stored-state encodings belong to deployments. Full plan acceptance is //! owned by `binding`, which also validates schemas, expressions and inputs. -use crate::summary_kernels::hll_sketch::HLL_PRECISIONS; -use crate::summary_kernels::weighted_frequency::{self, FrequencyAlgorithm}; + use crate::Error; -use planner_types::ir::schema::{ - ExactKind, ExactParams, FamilyCapability, FieldDataType as SummaryFamilyType, GroupingStrategy, - SketchAlgorithm, SketchParams, SketchStatistic, StatisticKind, SummaryUpdate, +use planner_types::ir::schema::FieldDataType as SummaryFamilyType; +use planner_types::summary_type::{ + ExactKind, ExactParams, SketchAlgorithm, SketchStatistic, StatisticKind, SummaryTypeCapability, + SummaryUpdate, }; /// Check the same contract used by `create_planner_accumulator` before a plan @@ -21,10 +21,9 @@ use planner_types::ir::schema::{ pub fn validate_summary_kernel( family: &SummaryFamilyType, input: &SummaryUpdate, - grouping: &GroupingStrategy, ) -> Result<(), String> { - if grouping != &GroupingStrategy::PerSubpopulationInstance { - return validate_hydra_cms(family, input, grouping); + if SummaryTypeCapability::shares_one_state_across_groups(family) { + return validate_hydra_cms(family, input); } let keyed = match family { SummaryFamilyType::ExactAggregate(kind, params) => { @@ -43,80 +42,8 @@ pub fn validate_summary_kernel( } input.item.is_some() } - SummaryFamilyType::Sketch(kind, layout) => { - if layout != grouping { - return Err("Planner family and operator grouping disagree".into()); - } - use SketchAlgorithm as A; - use SketchParams as P; - match (kind.algorithm(), kind.params()) { - // `asap_sketchlib` silently caps a larger `k`. - (A::Kll, P::Kll { k, .. }) - if (8..=planner_types::ir::schema::sketch::KLL_MAX_K).contains(k) => - { - false - } - (A::DDSketch, P::DDSketch { alpha }) - if alpha.is_finite() && *alpha > 0.0 && *alpha < 1.0 => - { - false - } - (A::Hll, P::Hll { precision, .. }) if HLL_PRECISIONS.contains(precision) => false, - (A::Kmv, P::Kmv { k, .. }) if *k >= crate::summary_kernels::kmv::KMV_MIN_K => false, - (A::Cms, P::Cms { width, depth }) if valid_matrix(*width, *depth) => true, - // The kernel's shape rules (an odd depth, for the median). - (A::CountSketch, P::CountSketch { width, depth }) - if valid_matrix(*width, *depth) - && weighted_frequency::check_shape( - FrequencyAlgorithm::CountSketch, - *width as usize, - *depth as usize, - ) - .is_ok() => - { - true - } - ( - A::CmsWithHeap, - P::CmsWithHeap { - width, - depth, - heap_size, - }, - ) - | ( - A::CountSketchWithHeap, - P::CountSketchWithHeap { - width, - depth, - heap_size, - }, - ) if valid_matrix(*width, *depth) && *heap_size > 0 => true, - ( - A::UnivMon, - P::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - }, - ) if crate::summary_kernels::univmon::check_shape( - *heap_size as usize, - *sketch_rows as usize, - *sketch_cols as usize, - usize::from(*layers), - ) - .is_ok() => - { - false - } - _ => { - return Err(format!( - "unsupported kernel or invalid parameters: {kind:?}" - )) - } - } + SummaryFamilyType::Sketch(_) => { + todo!("#1207 PR 6: check θ, which the exported node carries") } _ => return Err(format!("unsupported summary kernel {family:?}")), }; @@ -126,20 +53,13 @@ pub fn validate_summary_kernel( Ok(()) } -/// The only shared grouping with a kernel: Hydra over Count-Min. Each update +/// The only Hydra algorithm with a kernel: Hydra over Count-Min. Each update /// adds one item column's value to its group, with weight 1 (a row count, /// `UnitCount`), a positive integer constant (`PositiveConstant`) or a weight /// column proven non-negative (a Count-Min point count only overestimates /// when no weight is negative). -fn validate_hydra_cms( - family: &SummaryFamilyType, - input: &SummaryUpdate, - grouping: &GroupingStrategy, -) -> Result<(), String> { - use planner_types::ir::schema::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; - if !matches!(family, SummaryFamilyType::Sketch(_, layout) if layout == grouping) { - return Err("Planner family and operator grouping disagree".into()); - } +fn validate_hydra_cms(family: &SummaryFamilyType, input: &SummaryUpdate) -> Result<(), String> { + use planner_types::summary_type::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; hydra_cms_shape(family)?; if !matches!(input.item, Some(SummaryInputExpr::Column(_))) { return Err("HydraCms requires one item column".into()); @@ -182,47 +102,14 @@ pub(crate) fn constant_row_weight(weight: f64) -> Result Result<(usize, usize, usize, usize), String> { - use planner_types::ir::schema::{HydraKind, HydraParams}; - let SummaryFamilyType::Sketch( - kind, - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: - HydraParams::HydraCms { - width, - depth, - shared_rows, - shared_columns, - }, - }, - ) = family - else { - return Err("a shared-grouping (Hydra) family has no registered kernel".into()); + let SummaryFamilyType::Sketch(SketchAlgorithm::HydraCms) = family else { + return Err("a Hydra algorithm other than HydraCms has no registered kernel".into()); }; - if kind.algorithm() != &SketchAlgorithm::Cms - || kind.params() - != &(SketchParams::Cms { - width: *width, - depth: *depth, - }) - { - return Err("HydraCms family must be Count-Min with the Hydra width and depth".into()); - } - if !valid_matrix(*width, *depth) || !valid_matrix(*shared_columns, *shared_rows) { - return Err("invalid HydraCms dimensions".into()); - } - Ok(( - *shared_rows as usize, - *shared_columns as usize, - *width as usize, - *depth as usize, - )) + todo!("#1207 PR 6: the Hydra grid's sizes are θ, which the exported node carries") } /// A filtered `SummaryAgg` never runs at ingestion time. An ingestion-time @@ -230,7 +117,7 @@ pub(crate) fn hydra_cms_shape( /// none of the query's columns for the filter predicate to read, so only a /// query-time build can apply it. The ingestion-time compiler and the /// binding of any ingestion-timed `SummaryAgg` apply this check. The planner -/// applies the same rule: Stage 2 never offers `MatAtIngestion` for a +/// applies the same rule: Stage 2 never offers materializing at ingestion time for a /// filtered build (`IngestionPruneReason::Filtered`), and materialization /// timing rejects an explicit one (`FilteredSummaryAtIngestion`). This check /// guards plans that did not come from the planner. @@ -243,6 +130,7 @@ pub fn validate_ingestion_time_summary( } } +#[allow(dead_code)] // Used again in #1207 PR 6, with θ from the exported node. fn valid_matrix(width: u32, depth: u32) -> bool { // Construction uses the kernel's native row hashing, so no encoded-size // limit applies here. @@ -258,9 +146,9 @@ fn valid_matrix(width: u32, depth: u32) -> bool { /// occurrence), the update an unkeyed frequency family (HLL, UnivMon) builds /// from that column's values. pub(crate) fn unit_frequency_column( - update: &planner_types::ir::schema::SummaryUpdate, + update: &planner_types::summary_type::SummaryUpdate, ) -> Option<&planner_types::ir::scalar::ColumnRef> { - use planner_types::ir::schema::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; + use planner_types::summary_type::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; match (&update.item, &update.weight, &update.weight_domain) { ( Some(SummaryInputExpr::Column(column)), @@ -274,41 +162,24 @@ pub(crate) fn unit_frequency_column( } pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { - use planner_types::ir::schema::SketchAlgorithm as A; - if let SummaryFamilyType::Sketch(kind, grouping) = family { + use planner_types::summary_type::SketchAlgorithm as A; + if let SummaryFamilyType::Sketch(kind) = family { // Plain Count-Min is native as stored state only: it merges and reads // its bare count, but the DAG does not build it from rows. HydraCms // is built by the all-groups summary build. - if let (A::Cms, SketchParams::Cms { width, depth }) = (kind.algorithm(), kind.params()) { - if grouping != &GroupingStrategy::PerSubpopulationInstance { - return hydra_cms_shape(family).map(|_| ()).map_err(Error::Invalid); - } - return if valid_matrix(*width, *depth) { - Ok(()) - } else { - Err(Error::Invalid( - "invalid Count-Min dimensions or grouping strategy".into(), - )) - }; + if let A::HydraCms = kind { + return hydra_cms_shape(family).map(|_| ()).map_err(Error::Invalid); + } + if let A::Cms = kind { + todo!("#1207 PR 6: check the Count-Min θ, which the exported node carries") } - if matches!(kind.algorithm(), A::CmsWithHeap | A::CountSketchWithHeap) { - let (_, width, depth, _) = weighted_frequency::WeightedFrequency::configuration(kind)?; - return if valid_matrix(width as u32, depth as u32) && grouping == &Default::default() { - Ok(()) - } else { - Err(Error::Invalid( - "invalid weighted frequency dimensions or grouping strategy".into(), - )) - }; + if matches!(kind, A::CmsWithHeap | A::CountSketchWithHeap) { + todo!("#1207 PR 6: check the weighted-frequency θ, which the exported node carries") } } match family { SummaryFamilyType::ExactAggregate(..) => {} - SummaryFamilyType::Sketch(kind, _) - if matches!( - kind.algorithm(), - A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon - ) => {} + SummaryFamilyType::Sketch(A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon) => {} _ => { return Err(Error::Invalid( "summary family has no native DAG state implementation".into(), @@ -317,10 +188,9 @@ pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { } crate::capability::validate_summary_kernel( family, - &planner_types::ir::schema::SummaryUpdate::column( + &planner_types::summary_type::SummaryUpdate::column( planner_types::ir::scalar::ColumnRef::SampleValue, ), - &Default::default(), ) .map_err(Error::Invalid) } @@ -354,16 +224,11 @@ pub(crate) const EVALUATED_READOUTS: &[(SketchAlgorithm, &[StatisticKind])] = &[ /// Whether the executor reads `readout` from a state of `family` whose /// family-capability table row answers it. fn evaluates(family: &SummaryFamilyType, readout: StatisticKind) -> bool { - use planner_types::ir::schema::HydraKind; match family { - SummaryFamilyType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => { - EVALUATED_READOUTS - .iter() - .any(|(family, readouts)| family == kind.algorithm() && readouts.contains(&readout)) - } - SummaryFamilyType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { - matches!(kind, HydraKind::HydraCms | HydraKind::HydraKll) - } + SummaryFamilyType::Sketch(SketchAlgorithm::HydraCms | SketchAlgorithm::HydraKll) => true, + SummaryFamilyType::Sketch(kind) => EVALUATED_READOUTS + .iter() + .any(|(family, readouts)| family == kind && readouts.contains(&readout)), _ => false, } } @@ -381,16 +246,7 @@ pub fn validate_sketch_evaluation( // HydraCms answers a group's item frequency. A group's total needs a // group-mass query asap_sketchlib's Hydra lacks. if readout == StatisticKind::Total - && matches!( - family, - SummaryFamilyType::Sketch( - _, - GroupingStrategy::SharedMultiSubpopulation { - kind: planner_types::ir::schema::HydraKind::HydraCms, - .. - } - ) - ) + && matches!(family, SummaryFamilyType::Sketch(SketchAlgorithm::HydraCms)) { return Err(Error::Invalid( "HydraCms does not answer a group total: asap_sketchlib's Hydra has no group-mass \ @@ -399,7 +255,7 @@ pub fn validate_sketch_evaluation( )); } if readout == StatisticKind::TopK - || !FamilyCapability::of(family).answers(query) + || !SummaryTypeCapability::of(family).answers(query) || !evaluates(family, readout) { return Err(Error::Invalid( @@ -422,7 +278,7 @@ pub fn validate_exact_evaluation( evaluation: &crate::summary_kernels::exact::ExactEvaluation, ) -> Result<(), Error> { validate_native_family(family)?; - use planner_types::ir::schema::ExactKind as E; + use planner_types::summary_type::ExactKind as E; let supported = matches!( family, SummaryFamilyType::ExactAggregate( diff --git a/crates/executor/src/evaluation.rs b/crates/executor/src/evaluation.rs index d294ec061..0284b8991 100644 --- a/crates/executor/src/evaluation.rs +++ b/crates/executor/src/evaluation.rs @@ -1,7 +1,7 @@ //! Evaluations over merged exact summary states. use crate::summary_kernels::exact::ExactAccumulator; use crate::AggregateCore; -use planner_types::ir::schema::ExactKind; +use planner_types::summary_type::ExactKind; use std::sync::Arc; fn merge_exact_states( @@ -58,7 +58,8 @@ pub fn exact_evaluation( #[cfg(test)] mod counter_tests { use super::*; - use planner_types::ir::schema::{ExactParams, FieldDataType as SummaryFamilyType}; + use planner_types::ir::schema::FieldDataType as SummaryFamilyType; + use planner_types::summary_type::ExactParams; fn counter(kind: ExactKind, params: ExactParams, keyed: bool) -> ExactAccumulator { ExactAccumulator::new(SummaryFamilyType::ExactAggregate(kind, params), keyed).unwrap() diff --git a/crates/executor/src/expressions/columnar.rs b/crates/executor/src/expressions/columnar.rs index 9c15bc96b..dbb37b317 100644 --- a/crates/executor/src/expressions/columnar.rs +++ b/crates/executor/src/expressions/columnar.rs @@ -180,6 +180,7 @@ fn literal(value: &ScalarValue) -> (Value, DataType) { ScalarValue::Float64(v) => (Value::Float64(*v), DataType::Float64), ScalarValue::Utf8(v) => (Value::Utf8(v.as_str().into()), DataType::Utf8), ScalarValue::Boolean(v) => (Value::Bool(*v), DataType::Bool), + ScalarValue::Timestamp(ms) => (Value::Timestamp(*ms), DataType::Timestamp), ScalarValue::Interval { months, days, diff --git a/crates/executor/src/expressions/planner.rs b/crates/executor/src/expressions/planner.rs index dc49634a6..349496419 100644 --- a/crates/executor/src/expressions/planner.rs +++ b/crates/executor/src/expressions/planner.rs @@ -71,6 +71,7 @@ pub(super) fn evaluate( ScalarValue::Utf8(value) => Value::Utf8(value.clone().into()), ScalarValue::Boolean(value) => Value::Bool(*value), ScalarValue::Null => Value::Null, + ScalarValue::Timestamp(ms) => Value::Timestamp(*ms), }), ScalarExpr::Cast { expr, to, .. } => { let value = evaluate(expr, row, schema, evaluation_time_ms, regexes)?; diff --git a/crates/executor/src/operators/aggregate/mod.rs b/crates/executor/src/operators/aggregate/mod.rs index 0feb88ddc..e914f171e 100644 --- a/crates/executor/src/operators/aggregate/mod.rs +++ b/crates/executor/src/operators/aggregate/mod.rs @@ -155,7 +155,7 @@ impl Operator { | AggIntent::Deriv | AggIntent::PredictLinear { .. } | AggIntent::Increase - | AggIntent::Count { .. } + | AggIntent::Count | AggIntent::Sum { col: None } | AggIntent::Avg { col: None } | AggIntent::Min { col: None } @@ -187,7 +187,7 @@ impl Operator { .collect::>(); fields.push(result_field( "value", - if matches!(intent, AggIntent::Count { .. }) { + if matches!(intent, AggIntent::Count) { DataType::Int64 } else { DataType::Float64 diff --git a/crates/executor/src/operators/aggregate/temporal.rs b/crates/executor/src/operators/aggregate/temporal.rs index 0ea8cedfd..b913fad41 100644 --- a/crates/executor/src/operators/aggregate/temporal.rs +++ b/crates/executor/src/operators/aggregate/temporal.rs @@ -122,7 +122,7 @@ pub(in crate::operators) fn window_value( .map(|v| Value::Float64(v * (end as f64 - start as f64) / 1000.)), AggIntent::Increase => rate(points, start, end, true) .map(|v| Value::Float64(v * (end as f64 - start as f64) / 1000.)), - AggIntent::Count { .. } => Some(Value::Int64( + AggIntent::Count => Some(Value::Int64( i64::try_from(points.len()).map_err(|_| Error::Invalid("count overflow".into()))?, )), AggIntent::Sum { .. } | AggIntent::Avg { .. } => { @@ -326,7 +326,7 @@ mod tests { values::Batch, }; use futures::StreamExt; - use planner_types::ir::properties::AccuracyTarget; + use planner_types::ir::schema::{ DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, }; @@ -366,12 +366,7 @@ mod tests { for (intent, expected) in [ (AggIntent::Rate, Value::Float64(2.)), (AggIntent::Increase, Value::Float64(4.)), - ( - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, - Value::Int64(3), - ), + (AggIntent::Count, Value::Int64(3)), ] { let batch = Batch::try_new( schema.clone(), diff --git a/crates/executor/src/operators/mod.rs b/crates/executor/src/operators/mod.rs index eb391edbd..cfdca4446 100644 --- a/crates/executor/src/operators/mod.rs +++ b/crates/executor/src/operators/mod.rs @@ -9,8 +9,9 @@ use crate::{ use futures::StreamExt; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{ - DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, SummaryUpdate, + DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, }; +use planner_types::summary_type::SummaryUpdate; use std::{collections::BTreeMap, sync::Arc}; pub(crate) mod common; use crate::expressions::ordered; @@ -228,8 +229,8 @@ impl Operator { self.kind, Kind::Evaluation { query: SummaryEvaluation::Exact(crate::summary_kernels::exact::ExactEvaluation { - statistic: planner_types::ir::schema::ExactKind::Rate - | planner_types::ir::schema::ExactKind::Increase, + statistic: planner_types::summary_type::ExactKind::Rate + | planner_types::summary_type::ExactKind::Increase, .. }), .. diff --git a/crates/executor/src/operators/series_window.rs b/crates/executor/src/operators/series_window.rs index 6d81f48df..b4452438d 100644 --- a/crates/executor/src/operators/series_window.rs +++ b/crates/executor/src/operators/series_window.rs @@ -98,7 +98,7 @@ impl Operator { | AggIntent::PredictLinear { .. } | AggIntent::Increase | AggIntent::Delta - | AggIntent::Count { .. } + | AggIntent::Count | AggIntent::Sum { col: None } | AggIntent::Avg { col: None } | AggIntent::Min { col: None } diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 273597d46..66ae98b6b 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -4,7 +4,7 @@ use crate::values::arrow_columns::ColumnDepth; use crate::values::StateColumn; use crate::StateColumnUpdater; use arrow::array::{Array, BooleanArray, Float64Array, TimestampMillisecondArray, UInt32Array}; -use planner_types::ir::schema::{FamilyCapability, MergeInputRule}; +use planner_types::summary_type::{MergeInputRule, SummaryTypeCapability}; use std::num::NonZeroU32; /// What each input row adds to its item's count in a keyed summary build @@ -24,7 +24,7 @@ pub enum RowWeight { /// A summary evaluation: a sketch query, or an exact evaluation with typed parameters. #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] pub enum SummaryEvaluation { - Sketch(planner_types::ir::schema::SketchStatistic), + Sketch(planner_types::summary_type::SketchStatistic), Exact(crate::summary_kernels::exact::ExactEvaluation), } @@ -40,58 +40,8 @@ impl Operator { row_weight: RowWeight, groups: Vec, ) -> Result { - use crate::summary_kernels::weighted_frequency::WeightedFrequency; - crate::values::validate_family(&family)?; - let SummaryFamilyType::Sketch(kind, _) = &family else { - return Err(invalid("keyed sketch required")); - }; - WeightedFrequency::configuration(kind)?; - validate_groups(&input, &groups)?; - if items.is_empty() { - return Err(invalid("keyed summary requires item columns")); - } - if let RowWeight::Column(weight) = row_weight { - if !matches!( - plain(&input, weight)?.0, - DataType::Float64 | DataType::Int64 - ) { - return Err(invalid("keyed summary weights must be Float64 or Int64")); - } - } - for &item in &items { - if !matches!( - plain(&input, item)?.0, - DataType::Utf8 - | DataType::Timestamp - | DataType::Int64 - | DataType::Float64 - | DataType::Bool - | DataType::Null - ) { - return Err(invalid("unsupported keyed summary identity type")); - } - } - let mut fields = groups - .iter() - .map(|&i| input.fields[i].clone()) - .collect::>(); - fields.push(SummaryField { - name: "state".into(), - dtype: family.clone(), - nullable: false, - table: None, - }); - Ok(Self { - kind: Kind::PerGroupKeyedSummaryBuild { - family, - items, - row_weight, - groups, - filter: None, - }, - inputs: vec![input], - output: schema(fields), - }) + let _ = (input, family, items, row_weight, groups); + todo!("#1207 PR 6: the weighted-frequency θ, which the exported node carries") } /// One summary state shared by all groups (a HydraCms grid), emitted as /// one row per group that reads its own part of it. Each row adds its @@ -177,29 +127,8 @@ impl Operator { k: usize, output: SchemaRef, ) -> Result { - use crate::summary_kernels::weighted_frequency::WeightedFrequency; - crate::values::validate_family(&field(&input, state)?.dtype)?; - let SummaryFamilyType::Sketch(kind, _) = &field(&input, state)?.dtype else { - return Err(invalid("keyed evaluation requires summary state")); - }; - let (_, _, _, capacity) = WeightedFrequency::configuration(kind)?; - if k > capacity || output.fields.len() <= input.fields.len() { - return Err(invalid("invalid keyed evaluation shape or capacity")); - } - if state + 1 != input.fields.len() - || output.fields[..state] != input.fields[..state] - || output.fields.last().unwrap().dtype != SummaryFamilyType::Plain(DataType::Float64) - { - return Err(invalid( - "keyed evaluation must preserve partitions and return a Float64 score", - )); - } - crate::values::validate_schema(&output)?; - Ok(Self { - kind: Kind::KeyedEvaluation { state, k }, - inputs: vec![input], - output, - }) + let _ = (input, state, k, output); + todo!("#1207 PR 6: the weighted-frequency capacity of θ, which the exported node carries") } /// One summary state per group, updated from the value of one input /// column per row (`time` orders counter samples): an exact aggregate, @@ -226,8 +155,8 @@ impl Operator { && matches!( family, SummaryFamilyType::ExactAggregate( - planner_types::ir::schema::ExactKind::Rate - | planner_types::ir::schema::ExactKind::Increase, + planner_types::summary_type::ExactKind::Rate + | planner_types::summary_type::ExactKind::Increase, _ ) ) @@ -237,7 +166,6 @@ impl Operator { crate::capability::validate_summary_kernel( &family, &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), ) .map_err(Error::Invalid)?; let mut fields = groups @@ -324,7 +252,7 @@ impl Operator { let mut fields = input.fields.clone(); let result_type = if matches!( fields[state].dtype, - SummaryFamilyType::ExactAggregate(planner_types::ir::schema::ExactKind::Count, _) + SummaryFamilyType::ExactAggregate(planner_types::summary_type::ExactKind::Count, _) ) || integral_count(family, &query) { DataType::Int64 @@ -337,8 +265,8 @@ impl Operator { && matches!( fields[state].dtype, SummaryFamilyType::ExactAggregate( - planner_types::ir::schema::ExactKind::Min - | planner_types::ir::schema::ExactKind::Max, + planner_types::summary_type::ExactKind::Min + | planner_types::summary_type::ExactKind::Max, _ ) ); @@ -356,8 +284,8 @@ impl Operator { /// distinct count as an integer item); a distinct count (KMV, HLL) also /// takes Utf8, and UnivMon Utf8 and Bool. fn accepts_update_type(family: &SummaryFamilyType, dtype: &DataType) -> bool { - use planner_types::ir::schema::{ExactKind, SketchAlgorithm as A}; - let SummaryFamilyType::Sketch(kind, _) = family else { + use planner_types::summary_type::{ExactKind, SketchAlgorithm as A}; + let SummaryFamilyType::Sketch(kind) = family else { return *dtype == DataType::Float64 || matches!( family, @@ -366,8 +294,8 @@ fn accepts_update_type(family: &SummaryFamilyType, dtype: &DataType) -> bool { }; match dtype { DataType::Float64 | DataType::Int64 => true, - DataType::Utf8 => matches!(kind.algorithm(), A::Kmv | A::Hll | A::UnivMon), - DataType::Bool => kind.algorithm() == &A::UnivMon, + DataType::Utf8 => matches!(kind, A::Kmv | A::Hll | A::UnivMon), + DataType::Bool => kind == &A::UnivMon, _ => false, } } @@ -381,7 +309,7 @@ impl Operator { let Kind::Evaluation { state, query: - SummaryEvaluation::Sketch(planner_types::ir::schema::SketchStatistic::Cardinality), + SummaryEvaluation::Sketch(planner_types::summary_type::SketchStatistic::Cardinality), } = &self.kind else { return Err(invalid( @@ -400,11 +328,11 @@ impl Operator { /// output is Int64 and whose updates have unit weight; execution rejects a /// non-integral total rather than rounding it. fn integral_count(family: &SummaryFamilyType, query: &SummaryEvaluation) -> bool { - matches!(family, SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::ir::schema::SketchAlgorithm::Cms) + matches!(family, SummaryFamilyType::Sketch(kind) + if kind == &planner_types::summary_type::SketchAlgorithm::Cms) && matches!( query, - SummaryEvaluation::Sketch(planner_types::ir::schema::SketchStatistic::PointCount { + SummaryEvaluation::Sketch(planner_types::summary_type::SketchStatistic::PointCount { value: None, .. }) @@ -917,7 +845,7 @@ async fn merge_summary( }; // Counter states join consecutive windows pairwise, which is exact only // in time order: order the inputs by their window start. - if FamilyCapability::of(family).merge_inputs == Some(MergeInputRule::DisjointInTime) { + if SummaryTypeCapability::of(family).merge_inputs == Some(MergeInputRule::DisjointInTime) { entries.sort_by_cached_key(|&entry| { let (states, index) = state_of(entry); match states.storage() { @@ -1153,8 +1081,8 @@ fn evaluate_states( }; if !matches!( evaluation.statistic, - planner_types::ir::schema::ExactKind::Rate - | planner_types::ir::schema::ExactKind::Increase + planner_types::summary_type::ExactKind::Rate + | planner_types::summary_type::ExactKind::Increase ) { return false; } @@ -1206,7 +1134,8 @@ fn evaluate_states( let value = summary .estimate(query) .map_err(|e| Error::Operator(e.to_string()))?; - if integral && *query == planner_types::ir::schema::SketchStatistic::Cardinality + if integral + && *query == planner_types::summary_type::SketchStatistic::Cardinality { // A distinct count is a whole number: the estimate // rounded to the nearest one. diff --git a/crates/executor/src/physical_planner/compiled.rs b/crates/executor/src/physical_planner/compiled.rs index 6e884928b..0744c4ba2 100644 --- a/crates/executor/src/physical_planner/compiled.rs +++ b/crates/executor/src/physical_planner/compiled.rs @@ -1,7 +1,7 @@ //! Reader-independent physical computation and checked deployment instantiation. use super::*; use crate::sources::{DataStore, Scan}; -use planner_types::ir::materialized_view::MaterializedView; +use planner_types::plan::materialized_view::MaterializedView; #[derive(Clone)] #[cfg_attr(any(test, feature = "test-support"), derive(serde::Serialize))] @@ -24,7 +24,7 @@ pub(super) struct MaterializedViewSlot { pub(super) view: MaterializedView, /// The view node's schema. pub(super) schema: SchemaRef, - /// The data its state summarizes ([`PhysicalASAPDAG::view_window`]). + /// The data its state summarizes (`crate::sources::view_windows`). pub(super) window: ScanWindow, } @@ -37,7 +37,7 @@ pub(super) struct RawDataSlot { pub(super) schema: SchemaRef, /// The `Scan`'s predicates, compiled over `schema`. pub(super) predicates: Vec, - /// The rows its readers read ([`PhysicalASAPDAG::scan_window`]). + /// The rows its readers read (`crate::sources::scan_windows`). pub(super) window: ScanWindow, } diff --git a/crates/executor/src/physical_planner/logical.rs b/crates/executor/src/physical_planner/logical.rs index fc4f78c91..4b4d58ff1 100644 --- a/crates/executor/src/physical_planner/logical.rs +++ b/crates/executor/src/physical_planner/logical.rs @@ -36,8 +36,6 @@ pub(super) fn restore(dag: &PhysicalASAPDAG) -> Result Result { preflight_depth(dag)?; - // A tumbling-window chain stands for its tumbling windows; the deployment binds each tumbling window's - // raw input, so it runs only once expanded. - if let Some(node) = dag.nodes.iter().find(|n| { + if dag.nodes.iter().any(|n| { matches!( n.payload, Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) ) }) { - return Err(invalid(format!( - "node {}: a SummaryTumblingWindowChain runs as its tumbling windows; expand it with \ - `asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains` before compiling", - node.id - ))); + todo!("#1207 PR 6: run a chain, and a TimeRange or TimeShift over it by window index") } // Helper IDs lie above the u32 range; a larger Planner ID could collide. if let Some(node) = dag.nodes.iter().find(|n| n.id as u64 > u64::from(u32::MAX)) { @@ -119,10 +113,10 @@ fn compile_internal( ( edge.consumer, match edge.role { - planner_types::ir::physical_export::EdgeRole::Left => 0, - planner_types::ir::physical_export::EdgeRole::Input => 1, - planner_types::ir::physical_export::EdgeRole::Right => 2, - planner_types::ir::physical_export::EdgeRole::ScalarRef => 3, + planner_types::plan::physical_asap_dag::EdgeRole::Left => 0, + planner_types::plan::physical_asap_dag::EdgeRole::Input => 1, + planner_types::plan::physical_asap_dag::EdgeRole::Right => 2, + planner_types::plan::physical_asap_dag::EdgeRole::ScalarRef => 3, }, ) }); @@ -165,13 +159,13 @@ fn compile_internal( } // Each view's and each `Scan`'s window, the DAG indexed once for all: // a materialized tumbling-window chain has a view per tumbling window. - let view_windows = dag.view_windows(views.iter().map(|view| view.node_id)); - let scan_windows = dag.scan_windows( + let view_windows = view_windows(dag, views.iter().map(|view| view.node_id)); + let scan_windows = scan_windows( + dag, dag.nodes .iter() .filter(|node| matches!(node.payload, Payload::NonASAP(NonASAPOp::Scan { .. }))) .map(|node| node.id), - &Default::default(), ); let compile_roots: BTreeSet = roots.iter().copied().collect(); let mut physical_dag = CompiledPhysicalDAG::new(roots.to_vec()); @@ -383,7 +377,7 @@ fn compile_internal( ))); } let reduction = match measure { - AggIntent::Count { .. } => Reduction::Count, + AggIntent::Count => Reduction::Count, AggIntent::Sum { .. } => Reduction::Sum(value), AggIntent::Avg { .. } => Reduction::Avg(value), AggIntent::Min { .. } => Reduction::Min(value), @@ -454,10 +448,9 @@ fn compile_internal( // A closed row must include either all source labels or the explicit // complete-label identity. Projected labels alone are insufficient. if let Payload::ASAP(ASAPOp::SummaryAgg { - family, + summary_type: family, input: update, reduction: PlannerReduction::PerEntity, - grouping, filter: None, .. }) = &node.payload @@ -485,7 +478,7 @@ fn compile_internal( "per-entity summary requires complete source identity", )); } - crate::capability::validate_summary_kernel(family, update, grouping) + crate::capability::validate_summary_kernel(family, update) .map_err(Error::Invalid)?; let value = match (frequency_item, &update.weight) { (Some(item), _) => item, @@ -661,17 +654,15 @@ fn compile_internal( /// Whether every summary state `state` merges was built from a relation's /// rows (a SQL table's), where a global aggregate has a row over no input /// row; not from a PromQL vector's, where it has none. -fn built_from_relation(state: &OperatorNode) -> bool { +fn built_from_relation(state: &std::rc::Rc) -> bool { match state.asap() { - Some(ASAPOp::SummaryMerge { children, .. }) => { - children.iter().all(|child| built_from_relation(child)) - } + Some(ASAPOp::SummaryMerge { children, .. }) => children.iter().all(built_from_relation), Some( ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, ) => child.result_kind == planner_types::ir::OperatorResultKind::Relation, // A filter keeping whole states reads them from their build. _ => planner_types::ir::operator::asap::selected_from(state) - .is_some_and(|states| built_from_relation(states)), + .is_some_and(|states| built_from_relation(&states)), } } @@ -887,7 +878,7 @@ fn temporal_evaluation_drops_name(node: &PhysicalASAPDAGNode) -> bool { /// one only when the family-capability table gives the family a sound merge. pub(crate) fn require_sound_merge(input: &SchemaRef) -> Result<(), Error> { let state = &input.fields[summary_column(input)?].dtype; - if FamilyCapability::of(state).merge_inputs.is_none() { + if SummaryTypeCapability::of(state).merge_inputs.is_none() { return Err(invalid(format!( "summary merge over {state:?} is unsupported: the family has no sound merge" ))); @@ -1009,7 +1000,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { let state = summary_column(input)?; - use planner_types::ir::schema::ExactKind as E; + use planner_types::summary_type::ExactKind as E; let statistic = match &input.fields[state].dtype { FieldDataType::ExactAggregate(kind, _) => match kind { E::Sum | E::Count | E::Min | E::Max | E::Rate | E::Increase => *kind, @@ -1145,7 +1136,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result Reduction::Count, + AggIntent::Count => Reduction::Count, AggIntent::Cardinality { cols, .. } => { Reduction::Cardinality(if cols.is_empty() { vec![column(None)?] @@ -1208,10 +1199,9 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result Err(invalid("value operation has no native implementation")), }, Payload::ASAP(ASAPOp::SummaryAgg { - family, + summary_type: family, input: update, reduction, - grouping, filter, .. }) => { @@ -1221,7 +1211,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result build.with_row_filter(expression(&filter.0, input)?), None => Ok(build), @@ -1236,9 +1226,9 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { let family = &input.fields[state].dtype; - if !FamilyCapability::merges_groups(family) { + if !SummaryTypeCapability::merges_groups(family) { return Err(invalid(format!( - "a rollup of {family:?} states: the family cannot merge different groups" + "a rollup of {family:?} states: the summary type cannot merge different groups" ))); } keys.clone() @@ -1288,13 +1278,14 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result Result { - if grouping != &planner_types::ir::schema::GroupingStrategy::PerSubpopulationInstance { - crate::capability::validate_summary_kernel(family, update, grouping) - .map_err(Error::Invalid)?; + if matches!( + family, + FieldDataType::Sketch(planner_types::summary_type::SketchAlgorithm::HydraCms) + ) { + crate::capability::validate_summary_kernel(family, update).map_err(Error::Invalid)?; let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid("an all-groups summary requires explicit groups")); }; @@ -1317,8 +1308,7 @@ fn summary_agg_build( crate::capability::unit_frequency_column(update), unkeyed_frequency_family(family), ) { - crate::capability::validate_summary_kernel(family, update, grouping) - .map_err(Error::Invalid)?; + crate::capability::validate_summary_kernel(family, update).map_err(Error::Invalid)?; let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid( "summary construction requires explicit grouping columns", @@ -1336,10 +1326,10 @@ fn summary_agg_build( let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid("keyed summary requires explicit partitions")); }; - if matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &planner_types::ir::schema::SketchAlgorithm::CmsWithHeap) + if matches!(family, FieldDataType::Sketch(kind) if kind == &planner_types::summary_type::SketchAlgorithm::CmsWithHeap) && !matches!( update.weight_domain, - planner_types::ir::schema::WeightDomain::NonNegative { .. } + planner_types::summary_type::WeightDomain::NonNegative { .. } ) { return Err(invalid("CMS requires a nonnegative weight contract")); @@ -1370,7 +1360,7 @@ fn summary_agg_build( groups(input, keys)?, ); } - crate::capability::validate_summary_kernel(family, update, grouping).map_err(Error::Invalid)?; + crate::capability::validate_summary_kernel(family, update).map_err(Error::Invalid)?; let SummaryInputExpr::Column(column) = &update.weight else { return Err(invalid( "summary update expression must be projected to a column", @@ -1405,10 +1395,10 @@ fn row_weight(input: &SchemaRef, weight: &SummaryInputExpr) -> Result bool { - matches!(family, FieldDataType::Sketch(kind, _) if matches!( - kind.algorithm(), - planner_types::ir::schema::SketchAlgorithm::UnivMon - | planner_types::ir::schema::SketchAlgorithm::Hll + matches!(family, FieldDataType::Sketch(kind) if matches!( + kind, + planner_types::summary_type::SketchAlgorithm::UnivMon + | planner_types::summary_type::SketchAlgorithm::Hll )) } fn summary_column(input: &SchemaRef) -> Result { @@ -1579,7 +1569,6 @@ fn local_scalar(expr: &ScalarExpr) -> Result PhysicalASAPDAG { PhysicalASAPDAG { @@ -1598,130 +1587,17 @@ mod compilation_tests { Err(Error::Invalid(_)) )); } - - /// `window_count` one-minute windows over one table `Scan`, the `i`-th - /// shifted `i` minutes back, each window a root (the shape of a - /// tumbling-window-expanded DAG), with the roots' node ids. - fn shifted_windows_over_one_scan(window_count: i64) -> (PhysicalASAPDAG, Vec) { - use planner_types::ir::operator::{Source, TimeShift}; - use planner_types::ir::physical_export::compile_physical_asap_workload; - use planner_types::ir::schema::{Field, Schema}; - use planner_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - Operator, QueryRoot, - }; - let minute_ms = 60_000; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let windows: Vec<_> = (0..window_count) - .map(|i| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: i * minute_ms, - at: None, - }, - child: std::rc::Rc::clone(&scan), - })) - .unwrap(); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(minute_ms as u64), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap() - }) - .collect(); - // One timing cache, so every window reads the one `Scan`. The cache is - // keyed by node address, so `windows` stays alive while it runs. - let mut cache = NodeToTimedNodeCache::new(); - let roots: Vec<_> = windows - .iter() - .map(|window| { - let timed = apply_materialization_timings( - window, - &MaterializationAssignment::default(), - &mut cache, - ) - .unwrap(); - QueryRoot::Operator(timed) - }) - .collect(); - let dag = compile_physical_asap_workload(&roots).unwrap(); - let roots = dag.roots.iter().map(|&id| id as NodeId).collect(); - (dag, roots) - } - - /// Three one-minute windows shifted 0, 1 and 2 minutes back over one - /// `Scan`: the `Scan` reads the union `(T - 3m, T]`, and each window's - /// `TimeRange` ends its shift before `T` and is one minute wide. - #[test] - fn shifted_windows_over_one_scan_compile_to_their_windows() { - let (dag, roots) = shifted_windows_over_one_scan(3); - let compiled = serde_json::to_value(compile(&dag, &[], &roots).unwrap()).unwrap(); - let nodes = compiled["nodes"].as_object().unwrap(); - let scan_windows: Vec<_> = nodes - .values() - .filter_map(|node| node.get("RawData")) - .map(|slot| slot["window"].clone()) - .collect(); - assert_eq!( - scan_windows, - [serde_json::json!({"Within": {"end_offset_ms": 0, "width_ms": 180_000}})] - ); - let mut time_ranges: Vec<(i64, i64)> = nodes - .values() - .filter_map(|node| node.pointer("/Operator/operator/kind/TimeRange")) - .map(|kind| { - ( - kind["end_offset_ms"].as_i64().unwrap(), - kind["width_ms"].as_i64().unwrap(), - ) - }) - .collect(); - time_ranges.sort(); - assert_eq!( - time_ranges, - [(0, 60_000), (60_000, 60_000), (120_000, 60_000)] - ); - } - - /// 20,000 shifted windows over one `Scan` compile in linear time. A - /// search over every node per window, and over every root per reader of - /// the `Scan`, made the compile quadratic in the number of windows - /// (#993): in a debug build, 10,000 windows compiled in 5.4 s and 20,000 - /// in over 10 s before, 1.4 s and 3.1 s after. - #[test] - #[ignore = "slow: run by hand"] - fn many_windows_over_one_scan_compile_in_linear_time() { - let (dag, roots) = shifted_windows_over_one_scan(20_000); - let started = Instant::now(); - compile(&dag, &[], &roots).unwrap(); - eprintln!("compiled 20,000 windows in {:?}", started.elapsed()); - } } /// Helpers for the Planner's own tests (`test-support`). #[cfg(feature = "test-support")] pub mod test_support { use super::*; - use planner_types::physical::{Materialization, Medium}; + use planner_types::stages::{Lifetime, StorageMedium}; use std::collections::HashMap; /// The materialized views at `nodes`, in that order: each node that is - /// not a view is marked `MatAtIngestion`, and it and every node below it + /// not a view is marked materialized at ingestion time, and it and every node below it /// run at ingestion time (their outputs and outgoing edges); then every /// view's `retention_ms` is set ([`PhysicalASAPDAG::set_retention`]) /// with no tumbling window or range-evaluation span. For tests that bind states at @@ -1743,7 +1619,10 @@ pub mod test_support { .iter() .filter(|node| { nodes.contains(&(node.id as NodeId)) - && !planner_types::test_support::is_materialized_view(node.materialization) + && !matches!( + node.lifetime, + planner_types::stages::Lifetime::AcrossEvaluations(_) + ) }) .map(|node| node.id as NodeId) .collect(); @@ -1757,7 +1636,7 @@ pub mod test_support { for node in &mut dag.nodes { let id = node.id as NodeId; if marked.contains(&id) { - node.materialization = Materialization::MatAtIngestion(Medium::Memory); + node.lifetime = Lifetime::AcrossEvaluations(StorageMedium::Memory); } if at_ingestion.contains(&id) { node.output_state.timing = ExecutionTiming::IngestionTime; diff --git a/crates/executor/src/physical_planner/promql_rows.rs b/crates/executor/src/physical_planner/promql_rows.rs index f4897d185..d08070667 100644 --- a/crates/executor/src/physical_planner/promql_rows.rs +++ b/crates/executor/src/physical_planner/promql_rows.rs @@ -29,9 +29,9 @@ pub fn decode_series_identity(encoded: &str) -> Result, } /// Resolve the row representation before candidate search; see -/// [`planner_types::ir::schema::schema_support::with_promql_series_identity`]. +/// [`planner_types::ir::schema::series_identity::with_promql_series_identity`]. pub fn with_series_identity(root: &Rc) -> Result, Error> { - planner_types::ir::schema::schema_support::with_promql_series_identity(root).map_err(invalid) + planner_types::ir::schema::series_identity::with_promql_series_identity(root).map_err(invalid) } /// Construct source rows only from full identities. The named label columns diff --git a/crates/executor/src/sources/data_store.rs b/crates/executor/src/sources/data_store.rs index 22cc88b50..62003366f 100644 --- a/crates/executor/src/sources/data_store.rs +++ b/crates/executor/src/sources/data_store.rs @@ -3,9 +3,9 @@ use crate::{ values::{Batch, SchemaRef}, Error, }; -use planner_types::ir::materialized_view::MaterializedView; use planner_types::ir::operator::Source; use planner_types::ir::ScalarExpr; +use planner_types::plan::materialized_view::MaterializedView; /// A deployment's raw data and stored materialized views, as one run reads /// them. The deployment passes one handle per run, and the handle presents @@ -50,7 +50,7 @@ pub struct RawDataRequest<'a> { /// /// - Query run at evaluation time `T`: the smallest range covering /// every window the `Scan`'s readers read - /// ([`PhysicalASAPDAG::scan_window`](planner_types::ir::physical_export::PhysicalASAPDAG::scan_window)); + /// ([`PhysicalASAPDAG::scan_window`](planner_types::plan::physical_asap_dag::PhysicalASAPDAG::scan_window)); /// `None` when a reader reads the `Scan` outside a window. /// - Ingestion run: the run's `(window_start_ms, window_end_ms]` when /// the `Scan` is read through windows, else `None`. Each @@ -70,7 +70,7 @@ pub struct RawDataRequest<'a> { pub struct MaterializedViewRequest<'a> { /// `(start, end]` in milliseconds: the interval of data the view's state /// summarizes in this run - /// ([`PhysicalASAPDAG::view_window`](planner_types::ir::physical_export::PhysicalASAPDAG::view_window)). + /// ([`PhysicalASAPDAG::view_window`](planner_types::plan::physical_asap_dag::PhysicalASAPDAG::view_window)). /// In a query run at `T` with view window `(T - o - w, T - o]`, it is /// that range; in an ingestion run, `(window_end_ms - w, window_end_ms]`. pub time_range_ms: (i64, i64), diff --git a/crates/executor/src/sources/memory.rs b/crates/executor/src/sources/memory.rs index 598c22397..8f362b458 100644 --- a/crates/executor/src/sources/memory.rs +++ b/crates/executor/src/sources/memory.rs @@ -1,5 +1,5 @@ use super::*; -use planner_types::ir::physical_export::PhysicalASAPNodeId; +use planner_types::plan::physical_asap_dag::PhysicalASAPNodeId; use std::collections::BTreeMap; /// An immutable in-memory [`DataStore`], for tests and examples: raw tables diff --git a/crates/executor/src/sources/mod.rs b/crates/executor/src/sources/mod.rs index d9ff969a9..290fc40c3 100644 --- a/crates/executor/src/sources/mod.rs +++ b/crates/executor/src/sources/mod.rs @@ -10,15 +10,16 @@ use crate::{ Error, }; use futures::{stream, StreamExt}; -use planner_types::ir::materialized_view::MaterializedView; use planner_types::ir::operator::Source; -use planner_types::ir::physical_export::ScanWindow; use planner_types::ir::schema::DataType; use planner_types::ir::ScalarExpr; +use planner_types::plan::materialized_view::MaterializedView; use std::sync::Arc; mod data_store; pub use data_store::{DataStore, MaterializedViewRequest, RawDataRequest}; +mod window; +pub(crate) use window::{scan_windows, time_range_window, view_windows, ScanWindow}; mod memory; pub use memory::InMemoryDataStore; diff --git a/crates/executor/src/sources/window.rs b/crates/executor/src/sources/window.rs new file mode 100644 index 000000000..191701007 --- /dev/null +++ b/crates/executor/src/sources/window.rs @@ -0,0 +1,195 @@ +//! The rows a slot of a compiled DAG reads, back from the evaluation time: +//! the executor derives each `Scan`'s and each materialized view's window +//! from the `TimeRange` and `TimeShift` nodes above or below it, and pushes +//! it into the data-store read. + +use std::collections::{HashMap, HashSet}; + +use planner_types::ir::operator::asap::ASAPOp; +use planner_types::ir::operator::node::Operator; +use planner_types::ir::operator::non_asap::NonASAPOp; +use planner_types::plan::physical_asap_dag::{ + PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload, +}; + +/// Which rows a slot reads, back from an evaluation time `T`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +pub(crate) enum ScanWindow { + /// The rows with time in `(T - end_offset_ms - width_ms, T - + /// end_offset_ms]`. + Within { end_offset_ms: i64, width_ms: i64 }, + /// Rows at any time. + Unbounded, +} + +impl ScanWindow { + /// `(start, end]` at evaluation time `evaluation_time_ms`; `None` when + /// unbounded. + pub(crate) fn bounds_ms(self, evaluation_time_ms: i64) -> Option<(i64, i64)> { + match self { + Self::Within { + end_offset_ms, + width_ms, + } => { + let end = evaluation_time_ms.saturating_sub(end_offset_ms); + Some((end.saturating_sub(width_ms), end)) + } + Self::Unbounded => None, + } + } + + /// The smallest window holding both. + fn union(self, other: Self) -> Self { + let ( + Self::Within { + end_offset_ms: a_end, + width_ms: a_width, + }, + Self::Within { + end_offset_ms: b_end, + width_ms: b_width, + }, + ) = (self, other) + else { + return Self::Unbounded; + }; + let end_offset_ms = a_end.min(b_end); + let start_back_ms = a_end + .saturating_add(a_width) + .max(b_end.saturating_add(b_width)); + Self::Within { + end_offset_ms, + width_ms: start_back_ms.saturating_sub(end_offset_ms), + } + } +} + +/// The rows the `TimeRange` `time_range` keeps, back from the evaluation +/// time `T`: `(T - s - w, T - s]` for a `TimeRange(w)` over `TimeShift`s +/// whose offsets sum to `s`. `Unbounded` when one of those shifts has an `@` +/// anchor, or `time_range` is not a `TimeRange`. +pub(crate) fn time_range_window<'a>( + time_range: PhysicalASAPNodeId, + payload_of: impl Fn(PhysicalASAPNodeId) -> Option<&'a PhysicalASAPOperatorPayload>, +) -> ScanWindow { + let Some(Operator::NonASAP(NonASAPOp::TimeRange { range, child, .. })) = payload_of(time_range) + else { + return ScanWindow::Unbounded; + }; + let mut end_offset_ms = 0i64; + let mut below = payload_of(*child); + while let Some(Operator::NonASAP(NonASAPOp::TimeShift { shift, child })) = below { + if shift.at.is_some() { + return ScanWindow::Unbounded; + } + end_offset_ms = end_offset_ms.saturating_add(shift.offset_ms); + below = payload_of(*child); + } + ScanWindow::Within { + end_offset_ms, + width_ms: i64::try_from(range.as_millis()).unwrap_or(i64::MAX), + } +} + +fn payloads_by_id( + dag: &PhysicalASAPDAG, +) -> HashMap { + dag.nodes + .iter() + .map(|node| (node.id, &node.payload)) + .collect() +} + +/// Each of `scans`' window, by id: the union of the windows its consumers +/// read through `TimeShift`s without an `@` anchor, each a `TimeRange(w)` +/// (`(T - s - w, T - s]`). `Unbounded` when a root, any other consumer, or +/// a shift with an `@` anchor reads the `Scan`, or nothing reads it. +pub(crate) fn scan_windows( + dag: &PhysicalASAPDAG, + scans: impl IntoIterator, +) -> HashMap { + let payloads = payloads_by_id(dag); + let mut consumers: HashMap> = HashMap::new(); + for edge in &dag.edges { + consumers + .entry(edge.producer) + .or_default() + .push(edge.consumer); + } + let roots: HashSet = dag.roots.iter().copied().collect(); + let window_of = |scan: PhysicalASAPNodeId| { + let mut window: Option = None; + let mut pending = vec![(scan, 0i64)]; + while let Some((id, end_offset_ms)) = pending.pop() { + if roots.contains(&id) { + return ScanWindow::Unbounded; + } + for &consumer in consumers.get(&id).into_iter().flatten() { + let width_ms = match payloads.get(&consumer) { + Some(Operator::NonASAP(NonASAPOp::TimeShift { shift, .. })) + if shift.at.is_none() => + { + pending.push((consumer, end_offset_ms.saturating_add(shift.offset_ms))); + continue; + } + Some(Operator::NonASAP(NonASAPOp::TimeRange { range, .. })) => { + range.as_millis() + } + Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) => todo!( + "#1207 PR 6: the scan window of a chain read through a TimeRange over it" + ), + _ => return ScanWindow::Unbounded, + }; + let read = ScanWindow::Within { + end_offset_ms, + width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), + }; + window = Some(window.map_or(read, |window| window.union(read))); + } + } + window.unwrap_or(ScanWindow::Unbounded) + }; + scans + .into_iter() + .map(|scan| (scan, window_of(scan))) + .collect() +} + +/// Each of `nodes`' window, by id: the union of the windows of every +/// `TimeRange` below it. `Unbounded` when no `TimeRange` is below it or one +/// of them is unbounded (a whole SQL table, an `@` anchor). +pub(crate) fn view_windows( + dag: &PhysicalASAPDAG, + nodes: impl IntoIterator, +) -> HashMap { + let payloads = payloads_by_id(dag); + let window_of = |node: PhysicalASAPNodeId| { + let mut seen = HashSet::new(); + let mut pending = vec![node]; + let mut window: Option = None; + while let Some(id) = pending.pop() { + if !seen.insert(id) { + continue; + } + let read = match payloads.get(&id) { + Some(Operator::NonASAP(NonASAPOp::TimeRange { .. })) => { + time_range_window(id, |id| payloads.get(&id).copied()) + } + Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) => todo!( + "#1207 PR 6: the window a chain read through a TimeRange over it summarizes" + ), + Some(payload) => { + pending.extend(payload.children().into_iter().copied()); + continue; + } + None => continue, + }; + window = Some(window.map_or(read, |window| window.union(read))); + } + window.unwrap_or(ScanWindow::Unbounded) + }; + nodes + .into_iter() + .map(|node| (node, window_of(node))) + .collect() +} diff --git a/crates/executor/src/summary_kernels/count_min_sketch.rs b/crates/executor/src/summary_kernels/count_min_sketch.rs index 9feb98b4b..023605a79 100644 --- a/crates/executor/src/summary_kernels/count_min_sketch.rs +++ b/crates/executor/src/summary_kernels/count_min_sketch.rs @@ -1,7 +1,7 @@ //! Count-Min Sketch frequency summary over `asap_sketchlib::CountMinF64`. use crate::{AggregateCore, KernelError}; use asap_sketchlib::{CountMinF64, CountMinSketch, DataInput}; -use planner_types::ir::schema::SketchStatistic; +use planner_types::summary_type::SketchStatistic; #[derive(Debug, Clone)] pub struct CountMinSketchAccumulator { diff --git a/crates/executor/src/summary_kernels/datasketches_kll.rs b/crates/executor/src/summary_kernels/datasketches_kll.rs index 80c40c1a9..459da5447 100644 --- a/crates/executor/src/summary_kernels/datasketches_kll.rs +++ b/crates/executor/src/summary_kernels/datasketches_kll.rs @@ -1,7 +1,7 @@ //! KLL quantile summary over `asap_sketchlib::Kll`. use crate::{AggregateCore, KernelError}; use asap_sketchlib::Kll; -use planner_types::ir::schema::SketchStatistic; +use planner_types::summary_type::SketchStatistic; #[derive(Clone)] pub struct DatasketchesKLLAccumulator { diff --git a/crates/executor/src/summary_kernels/dd_sketch.rs b/crates/executor/src/summary_kernels/dd_sketch.rs index 295eb8e5f..a14baa083 100644 --- a/crates/executor/src/summary_kernels/dd_sketch.rs +++ b/crates/executor/src/summary_kernels/dd_sketch.rs @@ -1,7 +1,7 @@ //! DDSketch quantile summary over `asap_sketchlib::DdSketch`. use crate::{AggregateCore, KernelError}; use asap_sketchlib::DdSketch; -use planner_types::ir::schema::SketchStatistic; +use planner_types::summary_type::SketchStatistic; #[derive(Debug, Clone)] pub struct DDSketchAccumulator { diff --git a/crates/executor/src/summary_kernels/exact.rs b/crates/executor/src/summary_kernels/exact.rs index ebfe857aa..71f627468 100644 --- a/crates/executor/src/summary_kernels/exact.rs +++ b/crates/executor/src/summary_kernels/exact.rs @@ -1,7 +1,8 @@ //! Exact summary state identified by Planner family, independent of keyed layout. use super::increase::IncreaseAccumulator; use crate::AggregateCore; -use planner_types::ir::schema::{ExactKind, ExactParams, FieldDataType as SummaryFamilyType}; +use planner_types::ir::schema::FieldDataType as SummaryFamilyType; +use planner_types::summary_type::{ExactKind, ExactParams}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -444,7 +445,7 @@ mod tests { fn the_planner_prices_an_exact_state_at_its_size() { assert_eq!( std::mem::size_of::() as u64, - planner_types::ir::schema::EXACT_ACCUMULATOR_STATE_BYTES + planner_types::summary_type::EXACT_ACCUMULATOR_STATE_BYTES ); } diff --git a/crates/executor/src/summary_kernels/factory.rs b/crates/executor/src/summary_kernels/factory.rs index e3345397f..1ecea9d0d 100644 --- a/crates/executor/src/summary_kernels/factory.rs +++ b/crates/executor/src/summary_kernels/factory.rs @@ -6,9 +6,8 @@ use crate::summary_kernels::{ HydraKllSketchAccumulator, }; use crate::AggregateCore; -use planner_types::ir::schema::{ - FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchParams, -}; +use planner_types::ir::schema::FieldDataType as SummaryFamilyType; +use planner_types::summary_type::{SketchAlgorithm, SketchParams}; /// Generate the clone-based `AccumulatorUpdater` methods for updaters whose /// inner `acc` field implements `Clone + AggregateCore`. @@ -462,28 +461,30 @@ fn cms_dims(params: &SketchParams) -> (usize, usize) { /// Construct the kernel declared by a Planner SummaryAgg. No deployment config /// tags participate in this dispatch and unsupported payloads are errors. +/// +/// `parameters` is θ, which the exported node carries +/// (`PhysicalASAPDAGNode::parameters`); `None` for an exact accumulator. pub fn create_planner_accumulator( family: &SummaryFamilyType, - input: &planner_types::ir::schema::SummaryUpdate, - grouping: &planner_types::ir::schema::GroupingStrategy, + parameters: Option<&SketchParams>, + input: &planner_types::summary_type::SummaryUpdate, ) -> Result, String> { if input.item.is_some() && matches!( input.weight_domain, - planner_types::ir::schema::WeightDomain::NonNegative { + planner_types::summary_type::WeightDomain::NonNegative { proof: - planner_types::ir::schema::NonNegativeWeightProof::ResetAwareCounterDerivative + planner_types::summary_type::NonNegativeWeightProof::ResetAwareCounterDerivative } ) { return Err("window-weighted summaries require typed DAG binding; integer heap updaters cannot consume rates".into()); } - crate::capability::validate_summary_kernel(family, input, grouping)?; - use planner_types::ir::schema::GroupingStrategy; - if grouping != &GroupingStrategy::PerSubpopulationInstance { + crate::capability::validate_summary_kernel(family, input)?; + if matches!(family, SummaryFamilyType::Sketch(SketchAlgorithm::HydraCms)) { return Err( - "shared grouping has one state for all groups; build it with `create_hydra_cms`".into(), + "HydraCms has one state for all groups; build it with `create_hydra_cms`".into(), ); } if matches!(family, SummaryFamilyType::ExactAggregate(..)) { @@ -494,15 +495,15 @@ pub fn create_planner_accumulator( )?, })); } - let SummaryFamilyType::Sketch(kind, family_grouping) = family else { + let SummaryFamilyType::Sketch(kind) = family else { return Err(format!("unsupported Planner summary family {family:?}")); }; - if family_grouping != grouping { - return Err("Planner family and operator grouping disagree".into()); - } - let updater: Box = match (kind.algorithm(), kind.params()) { + let Some(params) = parameters else { + return Err("a sketch summary needs its parameters θ".into()); + }; + let updater: Box = match (kind, params) { (SketchAlgorithm::Kll, SketchParams::Kll { k, .. }) => { - if *k > planner_types::ir::schema::sketch::KLL_MAX_K { + if *k > planner_types::summary_type::KLL_MAX_K { return Err("KLL k exceeds runtime bound".into()); } Box::new(KllAccumulatorUpdater::new( @@ -525,7 +526,7 @@ pub fn create_planner_accumulator( } (SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, _) => { let (algorithm, width, depth, capacity) = - WeightedFrequency::configuration(kind).map_err(|e| e.to_string())?; + WeightedFrequency::configuration(kind, params).map_err(|e| e.to_string())?; Box::new( WeightedFrequencyUpdater::new(algorithm, width, depth, capacity) .map_err(|e| e.to_string())?, @@ -557,7 +558,7 @@ pub fn create_planner_accumulator( }), _ => { return Err(format!( - "unsupported Planner algorithm/parameters: {kind:?}" + "unsupported Planner algorithm/parameters: {kind:?} {params:?}" )) } }; @@ -672,132 +673,6 @@ impl AccumulatorUpdater for UnivMonUpdater { } } -#[cfg(test)] -mod planner_parameter_regression { - use super::*; - use planner_types::ir::schema::{SketchKind, SummaryInputExpr, SummaryUpdate}; - - // Planner width is the bucket count; depth is the independent hash-row count. - #[test] - fn planner_sketch_dimensions_are_not_transposed() { - for (algorithm, params) in [ - ( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 128, - depth: 3, - }, - ), - ( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: 128, - depth: 3, - }, - ), - ( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 128, - depth: 3, - heap_size: 8, - }, - ), - ( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 128, - depth: 3, - heap_size: 8, - }, - ), - ] { - let family = - SummaryFamilyType::Sketch(SketchKind::new(algorithm, params), Default::default()); - let update = SummaryUpdate { - item: Some(SummaryInputExpr::Column( - planner_types::ir::scalar::ColumnRef::Named("host".into()), - )), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), - }; - let state = create_planner_accumulator(&family, &update, &Default::default()) - .unwrap() - .snapshot_accumulator(); - let dims = match algorithm { - SketchAlgorithm::Cms => { - let s = state - .as_any() - .downcast_ref::() - .unwrap(); - (s.inner.rows(), s.inner.cols()) - } - SketchAlgorithm::CountSketch - | SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketchWithHeap => { - let (width, depth, _) = state - .as_any() - .downcast_ref::() - .unwrap() - .shape(); - (depth, width) - } - _ => unreachable!(), - }; - assert_eq!(dims, (3, 128), "{algorithm:?}"); - } - } - - // HydraCms builds one shared grid shaped by its Hydra params; it is never - // a per-group updater, and its family must be the matching Count-Min. - #[test] - fn hydra_cms_builds_one_shared_grid() { - use planner_types::ir::schema::{ - GroupingStrategy, HydraKind, HydraParams, NonNegativeWeightProof, WeightDomain, - }; - let grouping = |width| GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width, - depth: 3, - shared_rows: 5, - shared_columns: 32, - }, - }; - let family = |width| { - SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 128, - depth: 3, - }, - ), - grouping(width), - ) - }; - assert_eq!( - create_hydra_cms(&family(128)).unwrap().shape(), - (5, 32, 128, 3) - ); - assert!(create_hydra_cms(&family(64)).is_err()); - let update = SummaryUpdate { - item: Some(SummaryInputExpr::Column( - planner_types::ir::scalar::ColumnRef::Named("host".into()), - )), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }; - assert!( - crate::capability::validate_summary_kernel(&family(128), &update, &grouping(128)) - .is_ok() - ); - assert!(create_planner_accumulator(&family(128), &update, &grouping(128)).is_err()); - } -} - #[cfg(test)] mod ddsketch_domain { use super::*; @@ -851,7 +726,7 @@ mod ddsketch_domain { for (q, exact) in expected { let estimate = updater .acc - .estimate(&planner_types::ir::schema::SketchStatistic::Quantile { q }) + .estimate(&planner_types::summary_type::SketchStatistic::Quantile { q }) .unwrap(); assert!( (estimate - exact).abs() <= 0.01 * f64::abs(exact), @@ -899,7 +774,7 @@ mod tests { assert!(updater.update_value(&Value::Utf8("x".into()), 0).is_err()); let median = updater .take_accumulator() - .estimate(&planner_types::ir::schema::SketchStatistic::Quantile { q: 0.5 }) + .estimate(&planner_types::summary_type::SketchStatistic::Quantile { q: 0.5 }) .unwrap(); assert_eq!(median, 3.0); } @@ -919,7 +794,7 @@ mod tests { } let estimate = updater .take_accumulator() - .estimate(&planner_types::ir::schema::SketchStatistic::Cardinality) + .estimate(&planner_types::summary_type::SketchStatistic::Cardinality) .unwrap(); assert_eq!(estimate, 60.0); } diff --git a/crates/executor/src/summary_kernels/hll_sketch.rs b/crates/executor/src/summary_kernels/hll_sketch.rs index 191cfe605..7144eb7e8 100644 --- a/crates/executor/src/summary_kernels/hll_sketch.rs +++ b/crates/executor/src/summary_kernels/hll_sketch.rs @@ -2,7 +2,7 @@ //! `HyperLogLogImpl`, at a precision chosen at run time. use crate::{AggregateCore, KernelError}; use asap_sketchlib::DataInput; -use planner_types::ir::schema::SketchStatistic; +use planner_types::summary_type::SketchStatistic; use registers::Registers; /// The precisions a state can be built at. diff --git a/crates/executor/src/summary_kernels/hydra_cms.rs b/crates/executor/src/summary_kernels/hydra_cms.rs index 4f835d991..7d60dbfb7 100644 --- a/crates/executor/src/summary_kernels/hydra_cms.rs +++ b/crates/executor/src/summary_kernels/hydra_cms.rs @@ -1,4 +1,4 @@ -//! Hydra over Count-Min Sketch (`HydraKind::HydraCms`) over `asap_sketchlib::Hydra`. +//! Hydra over Count-Min Sketch (`SketchAlgorithm::HydraCms`) over `asap_sketchlib::Hydra`. //! //! One `shared_rows × shared_columns` grid of Count-Min cells serves every //! group: a group hashes to one cell per row, and that cell's Count-Min @@ -10,7 +10,7 @@ //! does not offer, so it is not answered here. use crate::{AggregateCore, KernelError}; use asap_sketchlib::{input::HydraCounter, CountMinSketch, DataInput, FastPath, Hydra, Vector2D}; -use planner_types::ir::schema::SketchStatistic; +use planner_types::summary_type::SketchStatistic; use std::sync::Arc; /// The single Hydra key column: a group's whole label tuple is one diff --git a/crates/executor/src/summary_kernels/kmv.rs b/crates/executor/src/summary_kernels/kmv.rs index b6b6adc36..d201e1e5c 100644 --- a/crates/executor/src/summary_kernels/kmv.rs +++ b/crates/executor/src/summary_kernels/kmv.rs @@ -6,7 +6,7 @@ use crate::summary_kernels::factory::AccumulatorUpdater; use crate::{AggregateCore, KernelError}; use asap_sketchlib::{DataInput, Kmv}; -use planner_types::ir::schema::SketchStatistic; +use planner_types::summary_type::SketchStatistic; /// The smallest `k` the estimate `(k - 1) / U(k)` is defined for. pub const KMV_MIN_K: u32 = 2; diff --git a/crates/executor/src/summary_kernels/state_column_updater.rs b/crates/executor/src/summary_kernels/state_column_updater.rs index e059ab25c..65426fa6d 100644 --- a/crates/executor/src/summary_kernels/state_column_updater.rs +++ b/crates/executor/src/summary_kernels/state_column_updater.rs @@ -2,13 +2,14 @@ //! group: the code the `SummaryAgg` build runs row by row, exposed so a //! runtime that ingests sample by sample builds the same states. use super::exact::ScalarState; -use super::factory::{create_hydra_cms, create_planner_accumulator, AccumulatorUpdater}; +use super::factory::{create_hydra_cms, AccumulatorUpdater}; use super::hydra_cms::{hydra_group_name, hydra_item, hydra_weight, HydraCms, HydraCmsGroup}; use super::weighted_frequency::WeightedFrequency; use crate::values::{StateColumn, Value}; use crate::{AggregateCore, Error}; -use planner_types::ir::scalar::ColumnRef; -use planner_types::ir::schema::{ExactKind, FieldDataType, GroupingStrategy, SummaryUpdate}; + +use planner_types::ir::schema::FieldDataType; +use planner_types::summary_type::ExactKind; use std::sync::Arc; /// The open summary states of one family, one per group, updated one sample @@ -47,6 +48,7 @@ enum GroupStates { Sketches(Vec>), /// A keyed frequency sketch (Count-Min or Count Sketch with a heap) per /// group. + #[allow(dead_code)] // Used again in #1207 PR 6, with θ from the exported node. Keyed { /// An empty state of the family's shape, cloned for each new group /// (boxed: much larger than the other kinds' fields). @@ -73,31 +75,25 @@ impl StateColumnUpdater { /// `SummaryAgg` the plan builds one state per group of: an exact /// accumulator, a value sketch (KLL, DDSketch, HLL, KMV, UnivMon), a /// keyed frequency sketch with a heap, or a HydraCms grid shared by all - /// groups. Bad input for any other family. + /// groups. Bad input for any other summary type. pub fn new(family: &FieldDataType) -> Result { crate::values::validate_family(family)?; - let hydra = matches!( - family, - FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { .. }) - ); let states = match family { - FieldDataType::Sketch(..) if hydra => GroupStates::Hydra { - grid: Box::new(create_hydra_cms(family).map_err(Error::Invalid)?), - names: Vec::new(), - }, + FieldDataType::Sketch(planner_types::summary_type::SketchAlgorithm::HydraCms) => { + GroupStates::Hydra { + grid: Box::new(create_hydra_cms(family).map_err(Error::Invalid)?), + names: Vec::new(), + } + } FieldDataType::ExactAggregate(..) => GroupStates::Exact { empty: ScalarState::empty(family).map_err(Error::Invalid)?, states: Vec::new(), }, - FieldDataType::Sketch(kind, grouping) - if grouping == &GroupingStrategy::PerSubpopulationInstance - && WeightedFrequency::configuration(kind).is_ok() => - { - let (algorithm, width, depth, capacity) = WeightedFrequency::configuration(kind)?; - GroupStates::Keyed { - empty: Box::new(WeightedFrequency::new(algorithm, width, depth, capacity)?), - states: Vec::new(), - } + FieldDataType::Sketch( + planner_types::summary_type::SketchAlgorithm::CmsWithHeap + | planner_types::summary_type::SketchAlgorithm::CountSketchWithHeap, + ) => { + todo!("#1207 PR 6: the keyed frequency state of θ, which the exported node carries") } FieldDataType::Sketch(..) => { // Fails for a family with no per-group value kernel. @@ -125,12 +121,8 @@ impl StateColumnUpdater { } fn sketch_updater(family: &FieldDataType) -> Result, Error> { - create_planner_accumulator( - family, - &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), - ) - .map_err(Error::Invalid) + let _ = family; + todo!("#1207 PR 6: build it with `create_planner_accumulator` and θ from the exported node's `parameters`") } /// The number of groups. @@ -483,7 +475,7 @@ impl StateColumnUpdater { #[cfg(test)] mod tests { use super::*; - use planner_types::ir::schema::ExactParams; + use planner_types::summary_type::ExactParams; // An exact updater charges the buffer it holds (its capacity), the // same bytes its sealed column charges. diff --git a/crates/executor/src/summary_kernels/traits.rs b/crates/executor/src/summary_kernels/traits.rs index 4cde5ae1a..372adeac1 100644 --- a/crates/executor/src/summary_kernels/traits.rs +++ b/crates/executor/src/summary_kernels/traits.rs @@ -1,4 +1,4 @@ -use planner_types::ir::schema::SketchStatistic; +use planner_types::summary_type::SketchStatistic; pub type KernelError = Box; diff --git a/crates/executor/src/summary_kernels/univmon.rs b/crates/executor/src/summary_kernels/univmon.rs index 508e535cc..ce20bcf8f 100644 --- a/crates/executor/src/summary_kernels/univmon.rs +++ b/crates/executor/src/summary_kernels/univmon.rs @@ -3,7 +3,7 @@ use crate::AggregateCore; use asap_sketchlib::{DataInput, UnivMon}; use planner_types::ir::scalar::ColumnRef; -use planner_types::ir::schema::SketchStatistic; +use planner_types::summary_type::SketchStatistic; type Error = Box; diff --git a/crates/executor/src/summary_kernels/weighted_frequency.rs b/crates/executor/src/summary_kernels/weighted_frequency.rs index e0211c327..c87b52cce 100644 --- a/crates/executor/src/summary_kernels/weighted_frequency.rs +++ b/crates/executor/src/summary_kernels/weighted_frequency.rs @@ -96,10 +96,11 @@ impl std::fmt::Debug for WeightedFrequency { impl WeightedFrequency { pub(crate) fn configuration( - kind: &planner_types::ir::schema::SketchKind, + algorithm: &planner_types::summary_type::SketchAlgorithm, + params: &planner_types::summary_type::SketchParams, ) -> Result<(FrequencyAlgorithm, usize, usize, usize), Error> { - use planner_types::ir::schema::{SketchAlgorithm as A, SketchParams as P}; - let (algorithm, width, depth, capacity) = match (kind.algorithm(), kind.params()) { + use planner_types::summary_type::{SketchAlgorithm as A, SketchParams as P}; + let (algorithm, width, depth, capacity) = match (algorithm, params) { ( A::CmsWithHeap, P::CmsWithHeap { diff --git a/crates/executor/src/supported_summaries.rs b/crates/executor/src/supported_summaries.rs index 6ad48c3e1..c04acc01d 100644 --- a/crates/executor/src/supported_summaries.rs +++ b/crates/executor/src/supported_summaries.rs @@ -4,7 +4,7 @@ //! summary the executor would reject when it binds the plan. use crate::capability::EVALUATED_READOUTS; -use planner_types::ir::schema::SupportedSummary; +use planner_types::summary_type::SupportedSummary; /// Every sketch family this executor builds from rows and the statistics it /// answers from one state of it (`EVALUATED_READOUTS`). Exact accumulators @@ -12,8 +12,8 @@ use planner_types::ir::schema::SupportedSummary; pub fn supported_summary_families() -> Vec { EVALUATED_READOUTS .iter() - .map(|&(family, readouts)| SupportedSummary { - family, + .map(|&(summary_type, readouts)| SupportedSummary { + summary_type, readouts: readouts.to_vec(), }) .collect() @@ -24,15 +24,15 @@ mod tests { use super::*; use crate::capability::validate_sketch_evaluation; use crate::factory::create_planner_accumulator; - use crate::summary_kernels::weighted_frequency::WeightedFrequency; - use planner_types::ir::operator::operator_properties::{Reduction, Source}; + + use planner_types::ir::operator::arguments::{Reduction, Source}; use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - DataType, FamilyCapability, Field, FieldDataType, GroupingStrategy, Schema, - SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, StatisticKind, - SummaryInputExpr, SummaryUpdate, - }; + use planner_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use planner_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; + use planner_types::summary_type::{ + SketchAlgorithm, SketchStatistic, StatisticKind, SummaryInputExpr, SummaryTypeCapability, + SummaryUpdate, + }; const ALGORITHMS: [SketchAlgorithm; 9] = [ SketchAlgorithm::UnivMon, @@ -56,57 +56,8 @@ mod tests { StatisticKind::TopK, ]; - /// Valid parameters of `algorithm`, as the Planner sizes them. - fn params(algorithm: &SketchAlgorithm) -> SketchParams { - use SketchAlgorithm as A; - match algorithm { - A::UnivMon => SketchParams::UnivMon { - heap_size: 32, - sketch_rows: 3, - sketch_cols: 1024, - layers: 8, - distinct_failure_probability: None, - }, - A::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - A::Cms => SketchParams::Cms { - width: 64, - depth: 3, - }, - A::Hll => SketchParams::Hll { - precision: 12, - relative_error: None, - }, - A::DDSketch => SketchParams::DDSketch { alpha: 0.01 }, - A::CmsWithHeap => SketchParams::CmsWithHeap { - width: 64, - depth: 3, - heap_size: 10, - }, - A::Kmv => SketchParams::Kmv { - k: 1024, - relative_error: None, - }, - A::CountSketch => SketchParams::CountSketch { - width: 64, - depth: 3, - }, - A::CountSketchWithHeap => SketchParams::CountSketchWithHeap { - width: 64, - depth: 3, - heap_size: 10, - }, - other => unreachable!("{other:?} is not probed"), - } - } - fn per_instance(algorithm: &SketchAlgorithm) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new(*algorithm, params(algorithm)), - GroupingStrategy::PerSubpopulationInstance, - ) + FieldDataType::Sketch(*algorithm) } /// A statistic of kind `readout`. @@ -134,7 +85,7 @@ mod tests { /// Whether the factory builds the per-instance `family` from rows, with /// a value column or keyed items. fn factory_builds(family: &FieldDataType) -> bool { - let FieldDataType::Sketch(_, grouping) = family else { + let FieldDataType::Sketch(_) = family else { return false; }; let value = SummaryUpdate::column(ColumnRef::SampleValue); @@ -142,8 +93,8 @@ mod tests { item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), ..value.clone() }; - create_planner_accumulator(family, &value, grouping).is_ok() - || create_planner_accumulator(family, &keyed, grouping).is_ok() + create_planner_accumulator(family, None, &value).is_ok() + || create_planner_accumulator(family, None, &keyed).is_ok() } /// Whether the executor answers `readout` from `family`'s state: top-k @@ -151,9 +102,11 @@ mod tests { /// sketch evaluation, whose kernel must then read it from a built state. fn evaluation_answers(family: &FieldDataType, readout: StatisticKind) -> bool { match (readout, family) { - (StatisticKind::TopK, FieldDataType::Sketch(kind, _)) => { - WeightedFrequency::configuration(kind).is_ok() - } + (StatisticKind::TopK, FieldDataType::Sketch(kind)) => matches!( + kind, + planner_types::summary_type::SketchAlgorithm::CmsWithHeap + | planner_types::summary_type::SketchAlgorithm::CountSketchWithHeap + ), _ => { let accepted = validate_sketch_evaluation(family, &statistic(readout)).is_ok(); if accepted { @@ -167,7 +120,7 @@ mod tests { /// Whether the kernel of a per-instance `family` state built from one /// update reads `readout`. fn kernel_answers(family: &FieldDataType, readout: StatisticKind) -> bool { - let FieldDataType::Sketch(_, grouping) = family else { + let FieldDataType::Sketch(_) = family else { return false; }; let value = SummaryUpdate::column(ColumnRef::SampleValue); @@ -175,8 +128,8 @@ mod tests { item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), ..value.clone() }; - let Ok(mut updater) = create_planner_accumulator(family, &value, grouping) - .or_else(|_| create_planner_accumulator(family, &keyed, grouping)) + let Ok(mut updater) = create_planner_accumulator(family, None, &value) + .or_else(|_| create_planner_accumulator(family, None, &keyed)) else { return false; }; @@ -208,10 +161,9 @@ mod tests { .unwrap(); let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: scan, - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap(); @@ -238,9 +190,9 @@ mod tests { let executed = factory_builds(&family) && evaluation_answers(&family, readout); let declared = listed .iter() - .any(|s| &s.family == algorithm && s.readouts.contains(&readout)); + .any(|s| &s.summary_type == algorithm && s.readouts.contains(&readout)); assert_eq!(declared, executed, "{algorithm:?} {readout:?}"); - let answered = FamilyCapability::of(&family).answers(&statistic(readout)); + let answered = SummaryTypeCapability::of(&family).answers(&statistic(readout)); assert_eq!( ir_builds_estimate(&family, readout), answered, diff --git a/crates/executor/src/values/batch.rs b/crates/executor/src/values/batch.rs index 2bc7e0284..77aa4e5fc 100644 --- a/crates/executor/src/values/batch.rs +++ b/crates/executor/src/values/batch.rs @@ -609,9 +609,8 @@ impl Batch { mod tests { use super::*; use crate::summary_kernels::DDSketchAccumulator; - use planner_types::ir::schema::{ - DataType, Field, Schema, SketchAlgorithm, SketchKind, SketchParams, - }; + use planner_types::ir::schema::{DataType, Field, Schema}; + use planner_types::summary_type::SketchAlgorithm; fn field(name: &str, dtype: DataType, nullable: bool) -> Field { Field { @@ -774,13 +773,7 @@ mod tests { } fn dd_family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - Default::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::DDSketch) } // Rows sharing a state keep sharing one state; filtering rows shares the @@ -861,7 +854,7 @@ mod tests { #[test] fn a_batch_from_columns_holds_sealed_states_and_labels() { use crate::StateColumnUpdater; - use planner_types::ir::schema::{ExactKind, ExactParams}; + use planner_types::summary_type::{ExactKind, ExactParams}; let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); let mut updater = StateColumnUpdater::new(&family).unwrap(); updater.ensure_groups(2).unwrap(); @@ -908,7 +901,7 @@ mod tests { .as_any() .downcast_ref::() .unwrap() - .evaluation(planner_types::ir::schema::ExactKind::Sum, None, None) + .evaluation(planner_types::summary_type::ExactKind::Sum, None, None) .unwrap() .unwrap() }; @@ -1004,7 +997,7 @@ mod tests { #[test] fn a_selective_filter_compacts_its_state_column() { use crate::StateColumnUpdater; - use planner_types::ir::schema::{ExactKind, ExactParams}; + use planner_types::summary_type::{ExactKind, ExactParams}; let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); let mut updater = StateColumnUpdater::new(&family).unwrap(); updater.ensure_groups(10).unwrap(); @@ -1051,7 +1044,7 @@ mod tests { .as_any() .downcast_ref::() .unwrap() - .evaluation(planner_types::ir::schema::ExactKind::Sum, None, None) + .evaluation(planner_types::summary_type::ExactKind::Sum, None, None) .unwrap() .unwrap() }) @@ -1064,7 +1057,7 @@ mod tests { #[test] fn concat_of_different_exact_columns_keeps_records() { use crate::StateColumnUpdater; - use planner_types::ir::schema::{ExactKind, ExactParams}; + use planner_types::summary_type::{ExactKind, ExactParams}; let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); let schema = Arc::new(Schema { fields: vec![Field { @@ -1122,7 +1115,7 @@ mod tests { .as_any() .downcast_ref::() .unwrap() - .evaluation(planner_types::ir::schema::ExactKind::Sum, None, None) + .evaluation(planner_types::summary_type::ExactKind::Sum, None, None) .unwrap() .unwrap() }) @@ -1134,22 +1127,11 @@ mod tests { /// exact sums. fn state_batch(count: u32, exact: bool) -> Batch { use crate::StateColumnUpdater; - use planner_types::ir::schema::{ - ExactKind, ExactParams, SketchAlgorithm, SketchKind, SketchParams, - }; + use planner_types::summary_type::{ExactKind, ExactParams, SketchAlgorithm}; let family = if exact { FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) } else { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::Kll) }; let mut updater = StateColumnUpdater::new(&family).unwrap(); updater.ensure_groups(count as usize).unwrap(); diff --git a/crates/executor/src/values/mod.rs b/crates/executor/src/values/mod.rs index a28ec8f44..6215fd15f 100644 --- a/crates/executor/src/values/mod.rs +++ b/crates/executor/src/values/mod.rs @@ -210,33 +210,11 @@ pub(super) fn validate_state( family: &SummaryFamilyType, state: &dyn AggregateCore, ) -> Result<(), Error> { - use crate::summary_kernels::{ - count_min_sketch::CountMinSketchAccumulator, datasketches_kll::DatasketchesKLLAccumulator, - dd_sketch::DDSketchAccumulator, exact::ExactAccumulator, hll_sketch::HllSketchAccumulator, - kmv::KmvAccumulator, univmon::UnivMonAccumulator, - }; - use planner_types::ir::schema::SketchParams; + use crate::summary_kernels::exact::ExactAccumulator; + validate_family(family)?; let valid = match family { - SummaryFamilyType::Sketch(kind, _) - if matches!( - kind.params(), - SketchParams::CmsWithHeap { .. } | SketchParams::CountSketchWithHeap { .. } - ) => - { - use crate::summary_kernels::weighted_frequency::WeightedFrequency; - let (algorithm, width, depth, capacity) = WeightedFrequency::configuration(kind)?; - state - .as_any() - .downcast_ref::() - .is_some_and(|state| { - state.algorithm() == algorithm && state.shape() == (width, depth, capacity) - }) - } - SummaryFamilyType::Sketch(_, grouping) - if grouping - != &planner_types::ir::schema::GroupingStrategy::PerSubpopulationInstance => - { + SummaryFamilyType::Sketch(planner_types::summary_type::SketchAlgorithm::HydraCms) => { let shape = crate::capability::hydra_cms_shape(family).map_err(Error::Invalid)?; state .as_any() @@ -247,49 +225,9 @@ pub(super) fn validate_state( .as_any() .downcast_ref::() .is_some_and(|s| s.family() == family && !s.is_keyed()), - SummaryFamilyType::Sketch(kind, _) => match kind.params() { - SketchParams::Kll { k, .. } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.inner.k() == *k as usize), - SketchParams::DDSketch { alpha } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.inner.alpha() == *alpha), - SketchParams::Hll { precision, .. } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.precision() == *precision), - SketchParams::Kmv { k, .. } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.inner.k == *k as usize), - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| { - s.dimensions() - == ( - *heap_size as usize, - *sketch_rows as usize, - *sketch_cols as usize, - usize::from(*layers), - ) - }), - SketchParams::Cms { width, depth } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| { - (s.inner.rows(), s.inner.cols()) == (*depth as usize, *width as usize) - }), - _ => false, - }, + SummaryFamilyType::Sketch(..) => { + todo!("#1207 PR 6: compare the state with θ, which the exported node carries") + } _ => false, }; if valid { @@ -343,50 +281,20 @@ pub(crate) fn plain(schema: &SchemaRef, column: usize) -> Result<(&DataType, boo mod weighted_state_tests { use super::*; use crate::summary_kernels::weighted_frequency::{FrequencyAlgorithm, WeightedFrequency}; - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::summary_type::SketchAlgorithm; // A state cannot acquire a different family or shape merely by relabeling its batch. #[test] fn weighted_state_family_and_shape_must_match() { - let cms = SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 32, - depth: 5, - heap_size: 8, - }, - ), - Default::default(), - ); - let cs = SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 32, - depth: 5, - heap_size: 8, - }, - ), - Default::default(), - ); + let cms = SummaryFamilyType::Sketch(SketchAlgorithm::CmsWithHeap); + let cs = SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap); let state = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 32, 5, 8).unwrap(); assert!(validate_state(&cs, &state).is_ok()); assert!(validate_state(&cms, &state).is_err()); let wrong_shape = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 64, 5, 8).unwrap(); assert!(validate_state(&cs, &wrong_shape).is_err()); - let even_depth = SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 32, - depth: 4, - heap_size: 8, - }, - ), - Default::default(), - ); + let even_depth = SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap); assert!(validate_family(&even_depth).is_err()); } } diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index ee3b4435b..7926c0416 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -186,7 +186,7 @@ fn cooperative_sort_preserves_ties_across_chunks() { // The integrated weighted-summary path obeys the same cooperative cancellation contract. #[test] fn weighted_summary_build_yields_within_a_batch() { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::summary_type::SketchAlgorithm; let input = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], @@ -222,17 +222,7 @@ fn weighted_summary_build_yields_within_a_batch() { Operator::source(input.clone(), vec![batch]).unwrap(), ) .unwrap(); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 3, - heap_size: 8, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); let operator = Operator::per_group_keyed_summary_build( input, family, diff --git a/crates/executor/tests/common/mod.rs b/crates/executor/tests/common/mod.rs index 857d76bf9..a3713995e 100644 --- a/crates/executor/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -1,82 +1,35 @@ #![allow(dead_code)] -use planner_types::ir::physical_export::PhysicalASAPDAG; -use planner_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, OperatorNode, - QueryRoot, -}; -use std::rc::Rc; -pub fn compile_physical_asap_dag( - root: &Rc, -) -> Result> { - let root = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::default(), - )?; - Ok(planner_types::ir::physical_export::compile_physical_asap_dag(&root)?) -} +use planner_types::ir::{OperatorNode, QueryRoot}; +use std::rc::Rc; /// The logical DAG the planner pipeline (`asap_planner::run_planning`) selects for the /// single query `root` at `accuracy`, over a continuously ingested source, /// with the built-in models. pub fn selected_dag( root: Rc, - accuracy: planner_types::ir::properties::AccuracyTarget, + accuracy: planner_types::accuracy::AccuracyTarget, ) -> Rc { let run = run_all_stages(root, accuracy); - let [QueryRoot::Operator(selected)] = run - .selected_plan - .as_deref() - .unwrap_or_else(|| panic!("a plan is selected: {:?}", run.selection)) - else { + let roots = match &run.selection { + Ok((plan, _)) => plan.query_roots().unwrap(), + Err(error) => panic!("a plan is selected: {error}"), + }; + let [QueryRoot::Operator(selected)] = roots.as_slice() else { panic!("one operator root"); }; Rc::clone(selected) } -/// Exercise each local implementation rule directly, keeping the other -/// computations at their default choice. A bounded solver archive need not -/// contain every algorithm that executor binding tests need to exercise. -pub fn computation_candidates(root: Rc) -> Vec> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .expect("Pass 1 accepts the query"); - let mut choices = vec![vec![0; inventory.targets.len()]]; - for (index, target) in inventory.targets.iter().enumerate() { - for choice in 1..target.choice_count() { - let mut selected = vec![0; inventory.targets.len()]; - selected[index] = choice; - choices.push(selected); - } - } - choices - .into_iter() - .map(|choice| { - let mut roots = compose_logical_candidate(&inventory, &choice) - .expect("a generated computation candidate composes"); - match roots.remove(0) { - QueryRoot::Operator(root) => root, - QueryRoot::Scalar(_) => panic!("one operator root"), - } - }) - .collect() -} - fn run_all_stages( root: Rc, - accuracy: planner_types::ir::properties::AccuracyTarget, + accuracy: planner_types::accuracy::AccuracyTarget, ) -> asap_planner::PlanningRun { use planner_types::workload::{DataArrival, DataWorkload}; let data = DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1_000.0, + max_out_of_order_ms: None, }, ..Default::default() }; diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index a78efe426..9d70317c0 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -1,5 +1,5 @@ //! Spatial heap weights come from a fresh instant vector, never sample history. -use crate::common; + use asap_executor::sources::InMemoryDataStore; use asap_executor::{ operators::Operator, @@ -10,11 +10,11 @@ use asap_executor::{ runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, }; -use common::compile_physical_asap_dag; use futures::{executor::block_on, StreamExt}; -use planner_types::ir::physical_export::PhysicalASAPOperatorPayload; -use planner_types::ir::properties::AccuracyTarget; +use planner_types::accuracy::AccuracyTarget; use planner_types::ir::schema::{DataType, *}; +use planner_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; +use planner_types::summary_type::*; use std::{collections::BTreeMap, sync::Arc}; fn schema() -> Arc { @@ -161,7 +161,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { SketchAlgorithm::CmsWithHeap, SketchAlgorithm::CountSketchWithHeap, ] { - let params = match algorithm { + let _params = match algorithm { SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { width: 2048, depth: 5, @@ -173,7 +173,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { heap_size: 100, }, }; - let family = FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()); + let family = FieldDataType::Sketch(algorithm); let build = Operator::per_group_keyed_summary_build( schema(), family, @@ -341,7 +341,7 @@ fn population_fixture_compiles_with_dynamic_identity() { topk, &["job"], ); - let logical = compile_physical_asap_dag(&selected).unwrap(); + let logical = planner_types::test_support::physical_asap_dag_at_query_time(&selected).unwrap(); let (source, raw_schema) = logical .nodes .iter() diff --git a/crates/executor/tests/current_timestamp.rs b/crates/executor/tests/current_timestamp.rs index a774d2289..415b7c0f9 100644 --- a/crates/executor/tests/current_timestamp.rs +++ b/crates/executor/tests/current_timestamp.rs @@ -10,11 +10,11 @@ use asap_executor::{ }; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::Source; -use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGNode}; use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ExprSemantics, ScalarValue}; use planner_types::ir::schema::{DataType, Field, Schema}; use planner_types::ir::{NonASAPOp, Predicate, ScalarExpr}; +use planner_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPDAGNode}; use std::sync::Arc; const MINUTE_MS: i64 = 60_000; @@ -55,7 +55,7 @@ fn scan_plan(predicates: Vec>) -> PhysicalASAPDAG { PhysicalASAPDAG { nodes: vec![PhysicalASAPDAGNode { id: 0, - payload: planner_types::ir::physical_export::PhysicalASAPOperatorPayload::NonASAP( + payload: planner_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload::NonASAP( NonASAPOp::Scan { source: Source::Table { table_ref: "http".into(), @@ -65,10 +65,12 @@ fn scan_plan(predicates: Vec>) -> PhysicalASAPDAG { }, ), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: schema(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }], edges: vec![], roots: vec![0], diff --git a/crates/executor/tests/deployment.rs b/crates/executor/tests/deployment.rs index 70804d9f4..11fd6e189 100644 --- a/crates/executor/tests/deployment.rs +++ b/crates/executor/tests/deployment.rs @@ -1,29 +1,17 @@ //! Exercise the public library without a backend server, store, or scheduler. -use asap_executor::planner::{ - ir::scalar::ColumnRef, - ir::schema::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, - }, -}; +use asap_executor::planner::ir::scalar::ColumnRef; +use asap_executor::planner::ir::schema::FieldDataType; +use asap_executor::planner::summary_type::{SketchAlgorithm, SketchParams, SummaryUpdate}; use asap_executor::{factory::create_planner_accumulator, AggregateCore}; -fn family(k: u32) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - GroupingStrategy::PerSubpopulationInstance, - ) +fn family(_k: u32) -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll) } fn build(values: &[f64]) -> Box { let mut operator = create_planner_accumulator( &family(512), + None, &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), ) .unwrap(); for (at, value) in values.iter().enumerate() { @@ -34,7 +22,7 @@ fn build(values: &[f64]) -> Box { } fn read(state: &dyn AggregateCore) -> f64 { state - .estimate(&asap_executor::planner::ir::schema::SketchStatistic::Quantile { q: 0.5 }) + .estimate(&asap_executor::planner::summary_type::SketchStatistic::Quantile { q: 0.5 }) .unwrap() } @@ -58,8 +46,8 @@ fn raw_partial_and_fully_precomputed_use_the_same_kernels() { fn invalid_kll_parameters_are_rejected_at_binding() { let result = create_planner_accumulator( &family(0), + None, &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), ); assert!(result.is_err()); } @@ -68,21 +56,11 @@ fn invalid_kll_parameters_are_rejected_at_binding() { // a packed-wire column-bit budget must not be imposed on this constructor. #[test] fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { - use asap_executor::planner::ir::schema::SummaryInputExpr; - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 1200, - depth: 55, - heap_size: 3, - }, - ), - Default::default(), - ); + use asap_executor::planner::summary_type::SummaryInputExpr; + let family = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap); let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let mut operator = create_planner_accumulator(&family, &update, &Default::default()).unwrap(); + let mut operator = create_planner_accumulator(&family, None, &update).unwrap(); operator .update_keyed(&["a".to_string()], 7.0, 1000) .unwrap(); @@ -99,14 +77,8 @@ fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { ); } -fn plain_count_sketch(width: u32, depth: u32) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width, depth }, - ), - Default::default(), - ) +fn plain_count_sketch(_width: u32, _depth: u32) -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::CountSketch) } fn host(i: usize) -> [String; 1] { @@ -126,10 +98,10 @@ fn build_weighted_frequency( family: &FieldDataType, updates: impl IntoIterator, ) -> Box { - use asap_executor::planner::ir::schema::SummaryInputExpr; + use asap_executor::planner::summary_type::SummaryInputExpr; let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let mut operator = create_planner_accumulator(family, &update, &Default::default()).unwrap(); + let mut operator = create_planner_accumulator(family, None, &update).unwrap(); for (at, (i, weight)) in updates.into_iter().enumerate() { operator.validate_single_input(weight).unwrap(); operator.update_keyed(&host(i), weight, at as i64).unwrap(); @@ -221,7 +193,7 @@ fn heap_sketch( depth: u32, heap_size: u32, ) -> FieldDataType { - let params = match algorithm { + let _params = match algorithm { SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { width, depth, @@ -233,7 +205,7 @@ fn heap_sketch( heap_size, }, }; - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) + FieldDataType::Sketch(algorithm) } // The factory's top-k sketches use the same per-row-seeded kernel: past 20 @@ -335,11 +307,11 @@ fn factory_and_dag_heap_states_are_interchangeable() { // validation, never silently dropped. #[test] fn count_min_heap_rejects_negative_inputs() { - use asap_executor::planner::ir::schema::SummaryInputExpr; + use asap_executor::planner::summary_type::SummaryInputExpr; let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = heap_sketch(SketchAlgorithm::CmsWithHeap, 64, 3, 4); - let operator = create_planner_accumulator(&family, &update, &Default::default()).unwrap(); + let operator = create_planner_accumulator(&family, None, &update).unwrap(); assert!(operator.validate_single_input(-1.0).is_err()); assert!(operator.validate_single_input(0.5).is_ok()); } @@ -401,11 +373,11 @@ fn plain_count_sketch_merge_equals_union_build() { // last good sum. #[test] fn keyed_update_overflow_is_an_error_not_a_panic() { - use asap_executor::planner::ir::schema::SummaryInputExpr; + use asap_executor::planner::summary_type::SummaryInputExpr; let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = plain_count_sketch(64, 3); - let mut operator = create_planner_accumulator(&family, &update, &Default::default()).unwrap(); + let mut operator = create_planner_accumulator(&family, None, &update).unwrap(); operator.validate_single_input(f64::MAX).unwrap(); operator.update_keyed(&host(0), f64::MAX, 0).unwrap(); let error = operator.update_keyed(&host(0), f64::MAX, 1).unwrap_err(); diff --git a/crates/executor/tests/deployment_computation.rs b/crates/executor/tests/deployment_computation.rs index 7fad67cef..72fe68ded 100644 --- a/crates/executor/tests/deployment_computation.rs +++ b/crates/executor/tests/deployment_computation.rs @@ -1,20 +1,20 @@ //! Planner-selected PromQL computation compiles from the timed DAG alone; //! the deployment supplies only raw rows at its time-series scans. -use crate::common; + use asap_executor::sources::InMemoryDataStore; use asap_executor::{ - physical_planner::{compile, promql_rows, test_support::views_at}, + physical_planner::{compile, promql_rows}, runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, }; -use common::{compile_physical_asap_dag, computation_candidates, selected_dag}; + use futures::{executor::block_on, StreamExt}; +use planner_types::accuracy::AccuracyTarget; use planner_types::ir::operator::maintained_population::PopulationStatistic; use planner_types::ir::operator::Source; -use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; -use planner_types::ir::properties::AccuracyTarget; use planner_types::ir::schema::*; -use planner_types::ir::ASAPOp; +use planner_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; + use planner_types::workload::*; use std::{collections::BTreeMap, rc::Rc, sync::Arc}; @@ -52,13 +52,6 @@ fn lower_with(query: &str, accuracy: AccuracyTarget) -> Rc PhysicalASAPDAG { - let expression = lower(query); - let root = promql_rows::with_series_identity(&expression).unwrap_or(expression); - compile_physical_asap_dag(&selected_dag(root, AccuracyTarget::exact())).unwrap() -} - /// The population sub-DAG of `query`, which computes `statistic` grouped by /// `grouping` over one selector. fn population_dag( @@ -70,7 +63,7 @@ fn population_dag( let selected = asap_logical_asap_optimization::test_support::population_over_selector( &root, statistic, grouping, ); - compile_physical_asap_dag(&selected).unwrap() + planner_types::test_support::physical_asap_dag_at_query_time(&selected).unwrap() } /// The time-series scans: each metric's raw data slot and its schema. @@ -265,433 +258,6 @@ fn global_population_aggregate_of_no_members_is_empty() { } } -// Scalar operands on either side apply to every grouped value, including negation. -#[test] -fn scalar_literal_arithmetic_applies_to_grouped_values() { - // sum_over_time over 5m per job: api = 4 + 1 + 7 + 2 = 14, db = 5. - for (query, expected) in [ - ( - "sum by (job) (sum_over_time(m[5m])) * 2", - reference(&[("api", 28.), ("db", 10.)]), - ), - ( - "100 - sum by (job) (sum_over_time(m[5m]))", - reference(&[("api", 86.), ("db", 95.)]), - ), - ( - "-sum by (job) (sum_over_time(m[5m]))", - reference(&[("api", -14.), ("db", -5.)]), - ), - ] { - assert_eq!( - run(&exact_dag(query), SAMPLES, 60_000).unwrap(), - expected, - "{query}" - ); - } -} - -// Grouped vectors match one-to-one on labels; unmatched groups are dropped and -// unchecked division by zero yields +Inf as in PromQL. -#[test] -fn grouped_vector_arithmetic_matches_labels() { - let samples: &[RawDataSample] = &[ - ("a", "api", "x", 10_000, 6.), - ("a", "api", "y", 20_000, 3.), - ("a", "db", "x", 10_000, 1.), - ("a", "web", "x", 10_000, 1.), - ("b", "api", "x", 10_000, 3.), - ("b", "db", "x", 10_000, 0.), - ("b", "cache", "x", 10_000, 1.), - ]; - let dag = - exact_dag("sum by (job) (sum_over_time(a[5m])) / sum by (job) (sum_over_time(b[5m]))"); - assert_eq!( - run(&dag, samples, 60_000).unwrap(), - reference(&[("api", 3.), ("db", f64::INFINITY)]) - ); -} - -// Exact observation counts finalize to the Float64 value PromQL declares, -// then roll up per job: api has 2 + 1 + 1 samples in 5m, db has 1. -#[test] -fn exact_count_finalizes_to_declared_float_value() { - let dag = exact_dag("sum by (job) (count_over_time(m[5m]))"); - assert_eq!( - run(&dag, SAMPLES, 60_000).unwrap(), - reference(&[("api", 4.), ("db", 1.)]) - ); -} - -/// Prometheus's `quantile` test series (`promql/promqltest/testdata/aggregators.test` -/// in Prometheus v0.314.0), one job per `test` label and one instance per -/// `point`, all sampled at 50 s. -const QUANTILE_SAMPLES: &[RawDataSample] = &[ - ("data", "two", "a", 50_000, 0.), - ("data", "two", "b", 50_000, 1.), - ("data", "three", "a", 50_000, 0.), - ("data", "three", "b", 50_000, 1.), - ("data", "three", "c", 50_000, 2.), - ("data", "uneven", "a", 50_000, 0.), - ("data", "uneven", "b", 50_000, 1.), - ("data", "uneven", "c", 50_000, 4.), - ("data", "nan", "a", 50_000, 0.), - ("data", "nan", "b", 50_000, 1.), - ("data", "nan", "c", 50_000, f64::NAN), -]; - -/// Asserts `(job, value)` rows equal `expected`, NaN equal to NaN and finite -/// values to within 1e-12 (Prometheus's own tests compare approximately too). -fn assert_rows(actual: &BTreeMap, expected: &[(&str, f64)], query: &str) { - let mut jobs: Vec<&str> = expected.iter().map(|(job, _)| *job).collect(); - jobs.sort(); - assert_eq!(actual.keys().collect::>(), jobs, "{query}"); - for (job, want) in expected { - let got = actual[*job]; - let same = if want.is_finite() { - (got - want).abs() <= 1e-12 * want.abs().max(1.) - } else { - got.to_bits() == want.to_bits() || (got.is_nan() && want.is_nan()) - }; - assert!(same, "{query}: {job} = {got}, want {want}"); - } -} - -// An exact instant `quantile by (G)` compiles to the grouped aggregate and -// returns Prometheus's values: linear interpolation between closest ranks, -// a NaN sample ordered first (so a rank between NaN and 0 is NaN), and ±Inf -// for φ outside [0, 1], NaN for a NaN φ. The 0.8, 0.2 and NaN rows are Prometheus's own expected -// values for `quantile without(point)` over the same series. -#[test] -fn exact_grouped_quantile_matches_prometheus() { - let (inf, nan) = (f64::INFINITY, f64::NAN); - for (query, expected) in [ - ( - "quantile by (job) (0.8, data)", - vec![("two", 0.8), ("three", 1.6), ("uneven", 2.8), ("nan", 0.6)], - ), - ( - "quantile by (job) (0.2, data)", - vec![("two", 0.2), ("three", 0.4), ("uneven", 0.4), ("nan", nan)], - ), - ( - "quantile by (job) (0, data)", - vec![("two", 0.), ("three", 0.), ("uneven", 0.), ("nan", nan)], - ), - ( - "quantile by (job) (1, data)", - vec![("two", 1.), ("three", 2.), ("uneven", 4.), ("nan", 1.)], - ), - ( - "quantile by (job) (-0.5, data)", - vec![ - ("two", -inf), - ("three", -inf), - ("uneven", -inf), - ("nan", -inf), - ], - ), - ( - "quantile by (job) (1.5, data)", - vec![("two", inf), ("three", inf), ("uneven", inf), ("nan", inf)], - ), - // Prometheus answers a NaN φ with NaN for every group. - ( - "quantile by (job) (NaN, data)", - vec![("two", nan), ("three", nan), ("uneven", nan), ("nan", nan)], - ), - // One group over every series: sorted [NaN, 0, 0, 0, 0, 1, 1, 1, 1, 2, 4], - // rank 0.5 * 10 = 5 → 1. - ("quantile(0.5, data)", vec![("", 1.)]), - ] { - let rows = run(&exact_dag(query), QUANTILE_SAMPLES, 60_000).unwrap(); - assert_rows(&rows, &expected, query); - } -} - -// With no input series the result is an empty vector, grouped or not, as in -// Prometheus; a group of one series returns that series' value for any φ in -// [0, 1]. -#[test] -fn exact_grouped_quantile_of_no_series_is_empty() { - for query in ["quantile by (job) (0.5, data)", "quantile(0.5, data)"] { - let rows = run(&exact_dag(query), &[], 60_000).unwrap(); - assert!(rows.is_empty(), "{query}: {rows:?}"); - } - let one: &[RawDataSample] = &[("data", "solo", "a", 50_000, 3.)]; - let query = "quantile by (job) (0.9, data)"; - assert_rows( - &run(&exact_dag(query), one, 60_000).unwrap(), - &[("solo", 3.)], - query, - ); -} - -/// `dag` with its Binary operator replaced by `kind`. -fn with_kind( - mut dag: PhysicalASAPDAG, - kind: planner_types::ir::operator::BinaryOpKind, - bool_result: bool, -) -> PhysicalASAPDAG { - for node in &mut dag.nodes { - if let PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { - operator, - return_bool, - .. - }) = &mut node.payload - { - operator.kind = kind.clone(); - *return_bool = bool_result; - } - } - dag -} - -// A comparison Binary over grouped values keeps the groups whose comparison -// holds, with their value, on either side of the literal; `bool` yields 1 or 0. -#[test] -fn grouped_comparisons_filter_or_return_bool() { - for (query, expected) in [ - ( - "sum by(job)(sum_over_time(m[5m])) > 10", - reference(&[("api", 14.)]), - ), - ( - "sum by(job)(sum_over_time(m[5m])) > bool 10", - reference(&[("api", 1.), ("db", 0.)]), - ), - ( - "10 > sum by(job)(sum_over_time(m[5m]))", - reference(&[("db", 5.)]), - ), - ] { - assert_eq!(run(&exact_dag(query), SAMPLES, 60_000).unwrap(), expected); - } -} - -// A `bool` comparison Binary over per-series evaluations matches one-to-one and -// drops the metric name; a filter keeps the surviving left value. -#[test] -fn per_series_comparisons_filter_or_return_bool() { - use planner_types::ir::operator::BinaryOpKind::*; - use planner_types::ir::scalar::CompareOpKind::*; - - let samples = counter("a", "api", 10., 10.) - .chain(counter("a", "db", 10., 10.)) - .chain(counter("b", "api", 5., 5.)) - .chain(counter("b", "db", 20., 20.)) - .collect::>(); - // rate: a{api} = a{db} = 50/300, b{api} = 25/300, b{db} = 100/300. - let dag = exact_dag("rate(a[5m]) / rate(b[5m])"); - assert_eq!( - run_series( - &with_kind(dag.clone(), Compare(Gt), false), - &samples, - 300_000 - ) - .unwrap(), - series(&[("api", "x", 50. / 300.)]) - ); - assert_eq!( - run_series(&with_kind(dag, Compare(Lt), true), &samples, 300_000).unwrap(), - series(&[("api", "x", 0.), ("db", "x", 1.)]) - ); -} - -/// [`execute`], returning per-series `(identity, value)` rows of the root, -/// with NaN-aware formatting for comparison. -fn run_series( - dag: &PhysicalASAPDAG, - samples: &[RawDataSample], - end: i64, -) -> Result { - let mut rows = BTreeMap::new(); - for batch in execute(dag, samples, end)? { - let schema = batch.schema(); - let column = |name: &str| schema.fields.iter().position(|f| f.name == name).unwrap(); - let (identity, value) = (column(promql_rows::SERIES_IDENTITY_COLUMN), column("value")); - for row in &batch.to_rows() { - let (Value::Utf8(identity), Value::Float64(value)) = (&row[identity], &row[value]) - else { - return Err(format!("unexpected row {row:?}")); - }; - assert!(rows.insert(identity.to_string(), *value).is_none()); - } - } - Ok(format!("{rows:?}")) -} - -fn series(pairs: &[(&str, &str, f64)]) -> String { - let rows = pairs - .iter() - .map(|(job, instance, value)| { - ( - format!(r#"{{"instance":"{instance}","job":"{job}"}}"#), - *value, - ) - }) - .collect::>(); - format!("{rows:?}") -} - -/// Five counter samples per series, one per minute up to 300s, at `base + step * i`. -fn counter( - metric: &'static str, - job: &'static str, - base: f64, - step: f64, -) -> impl Iterator { - (1..=5).map(move |i| (metric, job, "x", i * 60_000, base + step * (i - 1) as f64)) -} - -// avg_over_time over stored per-series sum/count state divides per series and -// drops the metric name, as Prometheus does. An overflowing stored sum fails -// the checked division instead of returning +Inf. -#[test] -fn per_series_average_divides_stored_sum_by_count() { - let dag = exact_dag("avg_over_time(m[5m])"); - assert_eq!( - run_series(&dag, SAMPLES, 60_000).unwrap(), - series(&[ - ("api", "a", 2.5), - ("api", "b", 7.), - ("api", "c", 2.), - ("db", "d", 5.) - ]) - ); - let huge: &[RawDataSample] = &[ - ("m", "api", "a", 10_000, 1.7e308), - ("m", "api", "a", 20_000, 1.7e308), - ]; - assert!(run_series(&dag, huge, 60_000).is_err()); -} - -// rate(a) / rate(b) matches series on their labels without the metric name. -// Unmatched series are dropped; x/0 is +Inf and 0/0 is NaN; an empty side -// yields an empty vector. -#[test] -fn per_series_rate_ratio_matches_prometheus() { - // Window (0, 300s]: first sample at 60s, extrapolated 60s to the start - // (durationToZero is exactly 60s too), so rate = (last - first) * 1.25 / 300. - // a{api} = 40 * 1.25 / 300, b{api} = 20 * 1.25 / 300, so the ratio is 2. - let samples = counter("a", "api", 10., 10.) - .chain(counter("a", "db", 10., 10.)) - .chain(counter("a", "web", 10., 10.)) - .chain(counter("a", "cache", 7., 0.)) - .chain(counter("b", "api", 5., 5.)) - .chain(counter("b", "web", 7., 0.)) - .chain(counter("b", "cache", 7., 0.)) - .chain(counter("b", "other", 5., 5.)) - .collect::>(); - let dag = exact_dag("rate(a[5m]) / rate(b[5m])"); - assert_eq!( - run_series(&dag, &samples, 300_000).unwrap(), - series(&[ - ("api", "x", 2.), - ("cache", "x", f64::NAN), - ("web", "x", f64::INFINITY), - ]) - ); - let only_a = samples - .iter() - .filter(|s| s.0 == "a") - .copied() - .collect::>(); - assert_eq!(run_series(&dag, &only_a, 300_000).unwrap(), series(&[])); -} - -// A literal operand applies to every stored per-series value, on either side, -// and drops the metric name. -#[test] -fn per_series_scalar_arithmetic_applies_to_stored_evaluations() { - let samples = counter("m", "api", 10., 10.).collect::>(); - // rate = 40 * 1.25 / 300 = 1/6. - for (query, expected) in [ - ("rate(m[5m]) * 2", 50. / 300. * 2.), - ("1 - rate(m[5m])", 1. - 50. / 300.), - // The stored sum evaluation keeps `__name__`; the arithmetic drops it. - ("sum_over_time(m[5m]) * 2", 150. * 2.), - ] { - assert_eq!( - run_series(&exact_dag(query), &samples, 300_000).unwrap(), - series(&[("api", "x", expected)]), - "{query}" - ); - } -} - -// A literal operand drops the metric name; series that then share a label set -// are an error, as in Prometheus, rather than duplicate output series. -#[test] -fn per_series_scalar_arithmetic_rejects_label_sets_equal_without_the_name() { - let dag = exact_dag("sum_over_time(m[5m]) * 2"); - let samples = counter("m", "api", 10., 10.) - .chain(counter("m", "api", 1., 1.).map(|s| (s.0, s.1, "y", s.3, s.4))) - .collect::>(); - let distinct = BTreeMap::from([("y", ("n", "y"))]); - assert!(execute_relabeled(&dag, &samples, 300_000, &distinct).is_ok()); - // Both series are then {instance="x",job="api"}, named m and n. - let equal = BTreeMap::from([("y", ("n", "x"))]); - let Err(error) = execute_relabeled(&dag, &samples, 300_000, &equal) else { - panic!("duplicate label sets were accepted"); - }; - assert!(error.contains("same labelset"), "{error}"); -} - -fn with_vector_match( - mut dag: PhysicalASAPDAG, - kind: planner_types::ir::operator::VectorMatchKind, - labels: &[&str], -) -> PhysicalASAPDAG { - for node in &mut dag.nodes { - if let PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { - operator, - .. - }) = &mut node.payload - { - operator.vector_match = Some(planner_types::ir::operator::VectorMatch { - kind: kind.clone(), - labels: labels.iter().map(|l| l.to_string()).collect(), - grouping: None, - }); - } - } - dag -} - -// `on` and `ignoring` reduce each side to the matching labels, which become -// the result's labels; a duplicate match group on either side is an error. -#[test] -fn per_series_vector_matching_follows_on_and_ignoring() { - use planner_types::ir::operator::VectorMatchKind; - let samples = counter("a", "api", 10., 10.) - .chain(counter("b", "api", 5., 5.)) - .chain(counter("b", "db", 5., 5.)) - .collect::>(); - let dag = exact_dag("rate(a[5m]) / rate(b[5m])"); - for (kind, labels) in [ - (VectorMatchKind::On, ["job"]), - (VectorMatchKind::Ignoring, ["instance"]), - ] { - let dag = with_vector_match(dag.clone(), kind, &labels); - assert_eq!( - run_series(&dag, &samples, 300_000).unwrap(), - format!( - "{:?}", - BTreeMap::from([(r#"{"job":"api"}"#.to_string(), 2.)]) - ) - ); - let mut duplicate = samples.clone(); - duplicate.extend(counter("b", "api", 5., 5.).map(|s| (s.0, s.1, "y", s.3, s.4))); - let error = run_series(&dag, &duplicate, 300_000).unwrap_err(); - assert!(error.contains("duplicate series"), "{error}"); - let mut duplicate = samples.clone(); - duplicate.extend(counter("a", "api", 5., 5.).map(|s| (s.0, s.1, "y", s.3, s.4))); - let error = run_series(&dag, &duplicate, 300_000).unwrap_err(); - assert!(error.contains("many-to-one"), "{error}"); - } -} - // Current-series sums and averages are compensated like Prometheus, and an // overflowing running sum does not turn the average into +Inf. #[test] @@ -728,98 +294,3 @@ fn population_sums_and_averages_are_compensated() { ); } } - -// A bare count over stored Count-Min state compiles to a Planner evaluation that -// returns the sketch's total update weight, including colliding items. -#[test] -fn stored_count_min_bare_count_compiles_to_a_evaluation() { - use asap_executor::summary_kernels::CountMinSketchAccumulator; - let accuracy = AccuracyTarget::max_error(0.02); - let root = lower_with("count(up)", accuracy.clone()); - let mut dag = computation_candidates(root) - .into_iter() - .find_map(|node| { - let dag = compile_physical_asap_dag(&node).ok()?; - let bare_count = dag.nodes.iter().any(|n| { - matches!( - &n.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryEstimate { - query: SketchStatistic::PointCount { value: None, .. }, - .. - }) - ) - }); - let count_min = dag.nodes.iter().any(|n| { - matches!(&n.payload, PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if kind.algorithm() == &SketchAlgorithm::Cms) - }); - (bare_count && count_min).then_some(dag) - }) - .expect("Planner lists a Count-Min candidate for count(up)"); - let state = dag - .nodes - .iter() - .find(|n| { - matches!( - n.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { .. }) - ) - }) - .unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = &state.payload - else { - unreachable!() - }; - let SketchParams::Cms { width, depth } = kind.params() else { - unreachable!() - }; - let (state_id, schema) = (state.id as u64, Arc::new(state.output_schema.clone())); - let (width, depth) = (*width, *depth); - let views = views_at(&mut dag, &[state_id]).unwrap(); - let program = compile(&dag, &views, &[dag.roots[0] as u64]).unwrap(); - let mut sketch = CountMinSketchAccumulator::new(depth as usize, width as usize).unwrap(); - sketch.update("a", 3.0); - sketch.update("b", 7.0); - let row = schema - .fields - .iter() - .map(|field| match &field.dtype { - FieldDataType::Plain(_) => panic!("unexpected stored column {field:?}"), - family => Value::Summary { - family: family.clone(), - state: Arc::new(sketch.clone()), - }, - }) - .collect(); - let batch = Batch::try_new(schema.clone(), vec![row]).unwrap(); - let store = InMemoryDataStore::default() - .with_materialized_view(state_id as usize, vec![batch]) - .unwrap(); - let physical_dag = program.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: 0, - }, - Limits::default(), - ) - .unwrap(); - let values = block_on(async { - let mut stream = physical_dag - .execute(program.roots(), context) - .unwrap() - .remove(0); - let mut values = Vec::new(); - while let Some(batch) = stream.next().await { - values.extend(batch.unwrap().to_rows().iter().map(|row| row[0].clone())); - } - values - }); - assert!( - matches!(values.as_slice(), [Value::Int64(10)]), - "{values:?}" - ); -} diff --git a/crates/executor/tests/exact_state_columns.rs b/crates/executor/tests/exact_state_columns.rs index 27cf337b7..25e89e23d 100644 --- a/crates/executor/tests/exact_state_columns.rs +++ b/crates/executor/tests/exact_state_columns.rs @@ -3,9 +3,8 @@ //! kernel objects (states read back from rows). use asap_executor::operators::{Operator, SummaryEvaluation}; use asap_executor::plan::PhysicalDAG; -use asap_executor::planner::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, -}; +use asap_executor::planner::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_executor::planner::summary_type::{ExactKind, ExactParams}; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::summary_kernels::exact::ExactEvaluation; use asap_executor::values::{Batch, SchemaRef, Value}; diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 7502bc10d..628737eab 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -9,15 +9,16 @@ use asap_executor::{ }; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::{GroupKeys, Reduction}; -use planner_types::ir::physical_export::{ - PhysicalASAPDAGNode, PhysicalASAPOperatorPayload as Payload, -}; use planner_types::ir::properties::*; use planner_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; use planner_types::ir::schema::DataType; use planner_types::ir::schema::*; use planner_types::ir::{ASAPOp, ExprSemantics, Predicate, ScalarExpr}; -use planner_types::physical::Materialization; +use planner_types::plan::physical_asap_dag::{ + PhysicalASAPDAGNode, PhysicalASAPOperatorPayload as Payload, +}; +use planner_types::stages::Lifetime; +use planner_types::summary_type::*; use std::{collections::BTreeMap, sync::Arc}; const WIDTH: u32 = 64; @@ -36,29 +37,8 @@ const ROWS: [(&str, &str, f64); 9] = [ ("db", "vacuum", 2.), ]; -fn grouping() -> GroupingStrategy { - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: WIDTH, - depth: 3, - shared_rows: 3, - shared_columns: SHARED_COLUMNS, - }, - } -} - fn family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: WIDTH, - depth: 3, - }, - ), - grouping(), - ) + FieldDataType::Sketch(SketchAlgorithm::HydraCms) } fn schema(fields: Vec) -> Schema { @@ -75,10 +55,12 @@ fn node(id: usize, payload: Payload, output: Schema) -> PhysicalASAPDAGNode { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: output, guarantee: None, retention_ms: None, + selected_physical_candidate: None, } } @@ -123,10 +105,9 @@ fn summary_agg(update: SummaryUpdate, filter: Option>) -> Physi 1, Payload::ASAP(ASAPOp::SummaryAgg { child: 0, - family: family(), + summary_type: family(), input: update, reduction: Reduction::Reduce(GroupKeys::by(vec![0])), - grouping: grouping(), filter, }), schema(vec![ @@ -205,7 +186,6 @@ fn try_run( 2, Payload::ASAP(ASAPOp::SummaryMerge { children: vec![1], - newest_tumbling_windows: None, group_by: None, }), (*state).clone(), @@ -420,7 +400,7 @@ fn constant_weight_above_i32_is_refused_at_validation() { }, ..unit_count() }; - let error = asap_executor::capability::validate_summary_kernel(&family(), &update, &grouping()) + let error = asap_executor::capability::validate_summary_kernel(&family(), &update) .expect_err("validation accepted a constant above i32::MAX"); assert!( error.contains("HydraCms") && error.contains("i32::MAX"), diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index 91f8bf032..8ceb8bdb2 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -10,14 +10,15 @@ use asap_executor::{ }; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::{GroupKeys, Reduction}; -use planner_types::ir::physical_export::{ - PhysicalASAPDAGNode, PhysicalASAPOperatorPayload as Payload, -}; use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::*; use planner_types::ir::ASAPOp; -use planner_types::physical::Materialization; +use planner_types::plan::physical_asap_dag::{ + PhysicalASAPDAGNode, PhysicalASAPOperatorPayload as Payload, +}; +use planner_types::stages::Lifetime; +use planner_types::summary_type::*; use std::sync::Arc; /// `(job, service)` rows: jobs are the groups, services the items. @@ -31,17 +32,7 @@ const ROWS: [(&str, &str); 6] = [ ]; fn heap_family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 1024, - depth: 3, - heap_size: 8, - }, - ), - GroupingStrategy::PerSubpopulationInstance, - ) + FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap) } fn schema(fields: Vec) -> SchemaRef { @@ -144,7 +135,7 @@ fn query_time_count_heap_binds_a_constant_row_weight() { id: 1, payload: Payload::ASAP(ASAPOp::SummaryAgg { child: 0, - family: heap_family(), + summary_type: heap_family(), input: SummaryUpdate { item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( ColumnRef::Named("service".into()), @@ -155,11 +146,11 @@ fn query_time_count_heap_binds_a_constant_row_weight() { }, }, reduction: Reduction::Reduce(GroupKeys::by(vec![0])), - grouping: GroupingStrategy::PerSubpopulationInstance, filter: None, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: (*schema(vec![ Field::plain("job", DataType::Utf8, false), Field { @@ -172,6 +163,7 @@ fn query_time_count_heap_binds_a_constant_row_weight() { .clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let build = compile_node(&node, std::slice::from_ref(&input)).unwrap(); let wire = serde_json::to_value(&build).unwrap(); @@ -247,17 +239,7 @@ fn int64_weight_column_equals_float64_column() { ), counts(limit as f64) ); - let count_sketch = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 1024, - depth: 3, - heap_size: 8, - }, - ), - GroupingStrategy::PerSubpopulationInstance, - ); + let count_sketch = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap); let signed = Operator::per_group_keyed_summary_build( int64.clone(), count_sketch, diff --git a/crates/executor/tests/main.rs b/crates/executor/tests/main.rs index 89abe7da8..ee0b768e5 100644 --- a/crates/executor/tests/main.rs +++ b/crates/executor/tests/main.rs @@ -17,7 +17,6 @@ mod physical_dag; mod physical_semantics; mod plan_properties; mod planspace_series_identity_heap; -mod precompute_candidates; mod precompute_population; mod promql_binary; mod promql_values; diff --git a/crates/executor/tests/node_ids.rs b/crates/executor/tests/node_ids.rs index 04a46fb54..961aa096d 100644 --- a/crates/executor/tests/node_ids.rs +++ b/crates/executor/tests/node_ids.rs @@ -1,7 +1,6 @@ //! Compilation rejects Planner node IDs outside the range its helper IDs avoid. -use crate::common; + use asap_executor::physical_planner::compile; -use common::compile_physical_asap_dag; use planner_types::ir::scalar::ScalarValue; use planner_types::ir::schema::{DataType, Field, Schema}; use planner_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; @@ -20,7 +19,7 @@ fn node_ids_above_u32_are_rejected() { child: values, })) .unwrap(); - let mut dag = compile_physical_asap_dag(&filter).unwrap(); + let mut dag = planner_types::test_support::physical_asap_dag_at_query_time(&filter).unwrap(); let root = dag.roots[0] as u64; assert!(compile(&dag, &[], &[root]).is_ok()); diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index bd22b97d5..9bc4f7fdd 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -8,15 +8,16 @@ use asap_executor::{ }; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::Source; -use planner_types::ir::physical_export::{ - EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, - PhysicalASAPOperatorPayload, WindowEdgeCompatibility, -}; -use planner_types::ir::schema::{DataType, ExactKind, ExactParams, Field, FieldDataType}; +use planner_types::ir::schema::{DataType, Field, FieldDataType}; use planner_types::ir::ASAPOp; use planner_types::ir::BinaryOperator; use planner_types::ir::NonASAPOp; use planner_types::ir::ScalarExpr; +use planner_types::plan::physical_asap_dag::{ + EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, + PhysicalASAPOperatorPayload, WindowEdgeCompatibility, +}; +use planner_types::summary_type::{ExactKind, ExactParams}; use std::sync::Arc; fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { Arc::new(planner_types::ir::schema::Schema { @@ -340,7 +341,7 @@ fn binding_rejects_unsupported_operations() { sum.schema(), 0, asap_executor::operators::SummaryEvaluation::Sketch( - planner_types::ir::schema::SketchStatistic::Quantile { q: 0.5 } + planner_types::summary_type::SketchStatistic::Quantile { q: 0.5 } ) ) .is_err()); @@ -350,19 +351,10 @@ fn binding_rejects_unsupported_operations() { // KLL is one family example: precomputation changes input sources, not operators. #[test] fn kll_raw_partial_and_precomputed_are_native_dags() { - use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::summary_type::SketchAlgorithm; let input = schema(&[("value", DataType::Float64, false)]); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 512, - failure_probability: None, - }, - ), - GroupingStrategy::PerSubpopulationInstance, - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let build = Operator::per_group_summary_build(input.clone(), family, 0, None, vec![]).unwrap(); let state = build.schema(); let build_range = |start: u32, end: u32| { @@ -447,7 +439,7 @@ fn kll_raw_partial_and_precomputed_are_native_dags() { state.clone(), 0, asap_executor::operators::SummaryEvaluation::Sketch( - planner_types::ir::schema::SketchStatistic::Quantile { q: 0.5 }, + planner_types::summary_type::SketchStatistic::Quantile { q: 0.5 }, ), ) .unwrap(), @@ -539,10 +531,12 @@ fn bind_physical_asap_dag_before_execution() { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*schema).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let mut dag = PhysicalASAPDAG { nodes: vec![ @@ -700,9 +694,11 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { payload, output_schema: (**schema).clone(), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let edge = |producer, consumer, role, schema: &asap_executor::values::SchemaRef| { PhysicalASAPDAGEdge::new( @@ -975,7 +971,7 @@ fn weighted_rate_topk_preserves_partitions_fractional_scores_and_evaluation_scop } } fn assert_weighted_rate_topk(count_sketch: bool) { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::summary_type::SketchAlgorithm; let raw = schema(&[ ("service", DataType::Utf8, false), @@ -1014,29 +1010,11 @@ fn assert_weighted_rate_topk(count_sketch: bool) { Some((0, 60_000)), ) .unwrap(); - let family = FieldDataType::Sketch( - SketchKind::new( - if count_sketch { - SketchAlgorithm::CountSketchWithHeap - } else { - SketchAlgorithm::CmsWithHeap - }, - if count_sketch { - SketchParams::CountSketchWithHeap { - width: 4096, - depth: 5, - heap_size: 8, - } - } else { - SketchParams::CmsWithHeap { - width: 4096, - depth: 5, - heap_size: 8, - } - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(if count_sketch { + SketchAlgorithm::CountSketchWithHeap + } else { + SketchAlgorithm::CmsWithHeap + }); let build = Operator::per_group_keyed_summary_build( rates.schema(), family, @@ -1136,10 +1114,12 @@ fn grouped_temporal_schema_compiles_and_executes_topk() { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*input).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -1226,7 +1206,8 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { id: 2, output_schema: (*schema).clone(), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::Join { @@ -1240,6 +1221,7 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { left: 0, right: 1, }), + selected_physical_candidate: None, }; let join = if certified { Operator::certified_semi_join(schema.clone(), schema.clone(), vec![(0, 0)]).unwrap() @@ -1331,7 +1313,8 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { id: 2, output_schema: (*input).clone(), output_state: ExecutionDataState::INGESTION_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { @@ -1345,6 +1328,7 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { lhs: 0, rhs: 1, }), + selected_physical_candidate: None, }; let binary = compile_node(&node, &[input.clone(), input.clone()]).unwrap(); for (right, expected) in [ diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 2831befe1..da811c8fb 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -10,12 +10,12 @@ use asap_executor::{ }; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::JoinKind; -use planner_types::ir::physical_export::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; use planner_types::ir::scalar::CompareOpKind; use planner_types::ir::schema::{DataType, Field, FieldDataType}; use planner_types::ir::NonASAPOp; use planner_types::ir::Predicate; use planner_types::ir::ScalarExpr; +use planner_types::plan::physical_asap_dag::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; use std::sync::Arc; fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { @@ -385,10 +385,12 @@ fn global_extrema_bind_with_planner_derived_schema() { child: 0, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*output).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let operator = compile_node(&node, std::slice::from_ref(&input)) .expect("global extremum should bind to its Planner schema"); @@ -554,19 +556,10 @@ fn boolean_truth_tables_agree_between_expression_paths() { // Partial/final execution must agree with one build for an uncompacted KLL population. #[test] fn kll_partial_merge_and_multiple_evaluations_preserve_population() { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::summary_type::SketchAlgorithm; let input = schema(&[("v", DataType::Float64, false)]); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 512, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let mut dag = PhysicalDAG::default(); for (id, range) in [(0, 0..64), (1, 64..128), (2, 0..128)] { let rows = range.map(|n| vec![Value::Float64(n as f64)]).collect(); @@ -610,7 +603,7 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { state.clone(), 0, asap_executor::operators::SummaryEvaluation::Sketch( - planner_types::ir::schema::SketchStatistic::Quantile { q }, + planner_types::summary_type::SketchStatistic::Quantile { q }, ), ) .unwrap(), @@ -650,7 +643,7 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { // single-build rank error: 60 tumbling windows of 1000 values at k = 200. #[test] fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams, SketchStatistic}; + use planner_types::summary_type::{SketchAlgorithm, SketchStatistic}; const TUMBLING_WINDOW_COUNT: u64 = 60; const ROWS_PER_TUMBLING_WINDOW: u64 = 1000; const K: u32 = 200; @@ -659,16 +652,7 @@ fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { let bound = 2.296 / f64::from(K).powf(0.9723); let n = (TUMBLING_WINDOW_COUNT * ROWS_PER_TUMBLING_WINDOW) as f64; let input = schema(&[("v", DataType::Float64, false)]); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: K, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let build = || { Operator::per_group_summary_build(input.clone(), family.clone(), 0, None, vec![]).unwrap() }; @@ -747,7 +731,7 @@ fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { #[test] fn counter_tumbling_windows_merge_in_any_order() { use asap_executor::summary_kernels::exact::ExactAccumulator; - use planner_types::ir::schema::{ExactKind, ExactParams}; + use planner_types::summary_type::{ExactKind, ExactParams}; let input = schema(&[ ("v", DataType::Float64, false), ("t", DataType::Timestamp, false), @@ -776,7 +760,7 @@ fn counter_tumbling_windows_merge_in_any_order() { .as_any() .downcast_ref::() .unwrap() - .evaluation(planner_types::ir::schema::ExactKind::Increase, None, None) + .evaluation(planner_types::summary_type::ExactKind::Increase, None, None) .unwrap() .unwrap() }; @@ -834,7 +818,7 @@ fn zero_column_output_obeys_memory_limit() { // Empty exact-state finalization must preserve ordinary global MIN/MAX null semantics. #[test] fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { - use planner_types::ir::schema::{ExactKind, ExactParams}; + use planner_types::summary_type::{ExactKind, ExactParams}; let input = schema(&[("v", DataType::Float64, false)]); for (kind, params) in [ @@ -935,7 +919,8 @@ fn planner_node( id: 1, payload, output_state: planner_types::ir::properties::ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: planner_types::ir::schema::Schema::new( fields .iter() @@ -949,6 +934,7 @@ fn planner_node( ), guarantee: None, retention_ms: None, + selected_physical_candidate: None, } } @@ -964,20 +950,16 @@ fn round_trip(operator: Operator) -> Operator { fn measure_filters_keep_groups_without_matching_rows() { use asap_executor::physical_planner::compile_node; use planner_types::ir::operator::{AggIntent, GroupKeys, Reduction as Plan}; - use planner_types::ir::properties::AccuracyTarget; + let (input, rows) = filtered_input(); let plain = |t: DataType| FieldDataType::Plain(t); let node = planner_node( PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Aggregate { reduction: Plan::Reduce(GroupKeys::by(vec![0])), measures: vec![ - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, + AggIntent::Count, AggIntent::Sum { col: Some(1) }, - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, + AggIntent::Count, ], output_names: vec!["c".into(), "s".into(), "all".into()], filters: vec![keep(), keep(), None], @@ -1031,22 +1013,12 @@ fn filtered_summary_build_keeps_empty_groups() { use asap_executor::physical_planner::compile_node; use planner_types::ir::operator::{GroupKeys, Reduction as Plan}; use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - ExactKind, ExactParams, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, - SummaryUpdate, - }; use planner_types::ir::ASAPOp; + use planner_types::summary_type::{ + ExactKind, ExactParams, SketchAlgorithm, SketchStatistic, SummaryUpdate, + }; let (input, rows) = filtered_input(); - let kll = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); let count = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); let cases = [ ( @@ -1069,10 +1041,9 @@ fn filtered_summary_build_keeps_empty_groups() { let build = planner_node( PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 0, - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::Named("x".into())), reduction: Plan::Reduce(GroupKeys::by(vec![0])), - grouping: Default::default(), filter: keep(), }), &[ @@ -1114,18 +1085,17 @@ fn filtered_summary_build_is_rejected_at_ingestion_time() { use planner_types::ir::operator::{GroupKeys, Reduction as Plan}; use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ExactKind, ExactParams, SummaryUpdate}; use planner_types::ir::ASAPOp; + use planner_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; let (input, _) = filtered_input(); let family = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); let build = |filter| { let mut node = planner_node( PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 0, - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::Named("x".into())), reduction: Plan::Reduce(GroupKeys::by(vec![0])), - grouping: Default::default(), filter, }), &[ @@ -1192,10 +1162,12 @@ fn exact_aggregate_node( child: 0, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*schema(&[(name, dtype, false)])).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, } } @@ -1205,7 +1177,7 @@ fn exact_aggregate_node( fn exact_frequency_intents_execute_typed_keys_and_empty_input() { use asap_executor::physical_planner::compile_node; use planner_types::ir::operator::AggIntent; - use planner_types::ir::properties::AccuracyTarget; + for (dtype, values) in [ ( DataType::Utf8, @@ -1227,18 +1199,12 @@ fn exact_frequency_intents_execute_typed_keys_and_empty_input() { let input = schema(&[("key", dtype, true)]); for (measure, name, expected) in [ ( - AggIntent::FrequencyL2 { - col: Some(0), - accuracy: AccuracyTarget::exact(), - }, + AggIntent::FrequencyL2 { col: Some(0) }, "frequency_l2", 8.0_f64.sqrt(), ), ( - AggIntent::FrequencyEntropy { - col: Some(0), - accuracy: AccuracyTarget::exact(), - }, + AggIntent::FrequencyEntropy { col: Some(0) }, "frequency_entropy", 1.0, ), @@ -1340,20 +1306,14 @@ fn exact_frequency_rejects_non_finite_identities() { fn exact_cardinality_binds_and_executes_typed_tuples() { use asap_executor::physical_planner::compile_node; use planner_types::ir::operator::AggIntent; - use planner_types::ir::properties::AccuracyTarget; + let input = schema(&[ ("key", DataType::Int64, true), ("tag", DataType::Utf8, true), ]); for (cols, expected) in [(vec![0], 2), (vec![0, 1], 3)] { - let node = exact_aggregate_node( - AggIntent::Cardinality { - cols, - accuracy: AccuracyTarget::exact(), - }, - "distinct", - DataType::Int64, - ); + let node = + exact_aggregate_node(AggIntent::Cardinality { cols }, "distinct", DataType::Int64); let operator = compile_node(&node, std::slice::from_ref(&input)).expect("exact distinct intent binds"); let rows = vec![ diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index 3b664a12a..6f1aa506d 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -7,15 +7,15 @@ use asap_executor::{ values::Batch, Error, }; -use planner_types::ir::materialized_view::MaterializedView; use planner_types::ir::operator::{GroupKeys, Source}; -use planner_types::ir::physical_export::{ - EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, - PhysicalASAPOperatorPayload, WindowEdgeCompatibility, -}; use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::schema::{DataType, FieldDataType, Schema}; use planner_types::ir::NonASAPOp; +use planner_types::plan::materialized_view::MaterializedView; +use planner_types::plan::physical_asap_dag::{ + EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, + PhysicalASAPOperatorPayload, WindowEdgeCompatibility, +}; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -49,10 +49,12 @@ fn sorted_scan(schema: &Schema, identity: &Source) -> PhysicalASAPDAG { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; PhysicalASAPDAG { nodes: vec![ @@ -128,102 +130,24 @@ fn blocking_inputs_over_a_raw_data_slot_are_finite() { assert_eq!(opens.load(Ordering::SeqCst), 1); } -/// A KLL kernel accepts `k` only up to the 26 602 `asap_sketchlib` caps it at. -#[test] -fn kll_kernel_rejects_a_k_above_the_library_cap() { - use asap_executor::capability::validate_summary_kernel; - use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryInputExpr, - SummaryUpdate, - }; - let grouping = GroupingStrategy::default(); - let update = SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: Default::default(), - }; - let kll = |k| { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - grouping.clone(), - ) - }; - assert!(validate_summary_kernel(&kll(26_602), &update, &grouping).is_ok()); - assert!(validate_summary_kernel(&kll(26_603), &update, &grouping).is_err()); -} - -/// A plain CountSketch reads the median of its rows, so plan validation and -/// kernel construction both require a positive odd depth. -#[test] -fn count_sketch_kernel_requires_an_odd_depth() { - use asap_executor::{capability::validate_summary_kernel, factory::create_planner_accumulator}; - use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryInputExpr, - SummaryUpdate, - }; - let grouping = GroupingStrategy::default(); - let mut update = SummaryUpdate::column(ColumnRef::SampleValue); - update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let count_sketch = |depth| { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width: 64, depth }, - ), - grouping.clone(), - ) - }; - for (depth, valid) in [(5, true), (4, false)] { - let family = count_sketch(depth); - assert_eq!( - validate_summary_kernel(&family, &update, &grouping).is_ok(), - valid - ); - assert_eq!( - create_planner_accumulator(&family, &update, &grouping).is_ok(), - valid - ); - } -} - // Kernel support must not be mistaken for executable native state/evaluation support. #[test] fn summary_capability_levels_are_distinct() { - use asap_executor::{ - capability::{validate_native_family, validate_sketch_evaluation, validate_summary_kernel}, - planner::ir::schema::SketchStatistic, + use asap_executor::capability::{ + validate_native_family, validate_sketch_evaluation, validate_summary_kernel, }; + use asap_executor::planner::summary_type::SketchStatistic; use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, - }; - let grouping = GroupingStrategy::default(); - let cms = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 64, - depth: 4, - }, - ), - grouping.clone(), - ); + use planner_types::summary_type::{SketchAlgorithm, SummaryUpdate}; + let cms = FieldDataType::Sketch(SketchAlgorithm::Cms); let update = SummaryUpdate { - item: Some(planner_types::ir::schema::SummaryInputExpr::Column( + item: Some(planner_types::summary_type::SummaryInputExpr::Column( ColumnRef::Named("host".into()), )), - weight: planner_types::ir::schema::SummaryInputExpr::Constant(1.0), + weight: planner_types::summary_type::SummaryInputExpr::Constant(1.0), weight_domain: Default::default(), }; - assert!(validate_summary_kernel(&cms, &update, &grouping).is_ok()); + assert!(validate_summary_kernel(&cms, &update).is_ok()); // Stored Count-Min state reads only its bare count natively. assert!(validate_native_family(&cms).is_ok()); let bare_count = SketchStatistic::PointCount { @@ -239,16 +163,7 @@ fn summary_capability_levels_are_distinct() { } ) .is_err()); - let kll = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 128, - failure_probability: None, - }, - ), - grouping, - ); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); assert!(validate_native_family(&kll).is_ok()); assert!(validate_sketch_evaluation(&kll, &SketchStatistic::Quantile { q: 1.5 }).is_err()); assert!(validate_sketch_evaluation(&kll, &SketchStatistic::Cardinality).is_err()); diff --git a/crates/executor/tests/planspace_series_identity_heap.rs b/crates/executor/tests/planspace_series_identity_heap.rs index 18919c26c..71bc6302a 100644 --- a/crates/executor/tests/planspace_series_identity_heap.rs +++ b/crates/executor/tests/planspace_series_identity_heap.rs @@ -2,11 +2,11 @@ //! the query's rows do not carry: the planner pipeline plans the logical DAG //! the frontend lowered, and a deployment prices heaps. use crate::common; -use common::{compile_physical_asap_dag, selected_dag}; +use common::selected_dag; use planner_types::ir::OperatorNode; use asap_executor::physical_planner::promql_rows::SERIES_IDENTITY_COLUMN; -use planner_types::ir::properties::AccuracyTarget; +use planner_types::accuracy::AccuracyTarget; use planner_types::workload::{ BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, @@ -44,7 +44,7 @@ fn lower(query: &str, accuracy: &AccuracyTarget) -> Rc { } fn carries_identity(root: &Rc) -> bool { - compile_physical_asap_dag(root) + planner_types::test_support::physical_asap_dag_at_query_time(root) .unwrap() .nodes .iter() diff --git a/crates/executor/tests/precompute_candidates.rs b/crates/executor/tests/precompute_candidates.rs deleted file mode 100644 index b540f8110..000000000 --- a/crates/executor/tests/precompute_candidates.rs +++ /dev/null @@ -1,363 +0,0 @@ -//! A grouped Rate→Sum answers the same whether the deployment stores the -//! Rate state or the grouped Sum state. -use crate::common; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::{ - factory::create_planner_accumulator, - physical_planner::{compile, test_support::views_at, CompiledPhysicalDAG}, - runtime::{Limits, RunContext, Scope}, - values::{Batch, Value}, -}; -use common::{compile_physical_asap_dag, computation_candidates, selected_dag}; -use futures::{executor::block_on, StreamExt}; -use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; -use planner_types::ir::properties::AccuracyTarget; -use planner_types::ir::schema::{DataType, *}; -use planner_types::ir::ASAPOp; -use planner_types::workload::*; -use std::{collections::BTreeMap, sync::Arc}; - -/// `sum by(job)(rate(m[1m]))` with its rows carrying the series identity. -fn grouped_rate_root() -> std::rc::Rc { - exact_promql_root("sum by(job)(rate(m[1m]))") -} - -/// `query`, exact, lowered with its rows carrying the series identity. -fn exact_promql_root(query: &str) -> std::rc::Rc { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy: AccuracyTarget::exact(), - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1000), - )), - }; - let root = asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap() - .remove(0) - .as_operator() - .cloned() - .expect("operator root"); - asap_executor::physical_planner::promql_rows::with_series_identity(&root).unwrap() -} - -fn grouped_rate() -> PhysicalASAPDAG { - compile_physical_asap_dag(&selected_dag(grouped_rate_root(), AccuracyTarget::exact())).unwrap() -} -fn run(plan: &CompiledPhysicalDAG, store: InMemoryDataStore, scope: Scope) -> Vec { - let dag = plan.instantiate(Arc::new(store)).unwrap(); - block_on(async { - let context = RunContext::new(scope, Limits::default()).unwrap(); - let mut output = dag.execute(plan.roots(), context).unwrap().remove(0); - let mut batches = vec![]; - while let Some(batch) = output.next().await { - batches.push((*batch.unwrap()).clone()); - } - batches - }) -} - -/// The root's output when the deployment stores the Rate `state` (bound to -/// `states`) and, with `stored`, also the output of node `stored`, which an -/// ingestion run over `ingestion` computes from the states. -fn answer( - dag: &PhysicalASAPDAG, - state: u64, - states: &Batch, - stored: Option, - ingestion: Scope, - query: Scope, -) -> Vec { - let mut dag = dag.clone(); - let views = views_at( - &mut dag, - &[state].into_iter().chain(stored).collect::>(), - ) - .unwrap(); - let state_store = || { - InMemoryDataStore::default() - .with_materialized_view(state as usize, vec![states.clone()]) - .unwrap() - }; - let root = [dag.roots[0] as u64]; - let Some(stored) = stored else { - return run(&compile(&dag, &views, &root).unwrap(), state_store(), query); - }; - let outputs = run( - &compile(&dag, &views[..1], &[stored]).unwrap(), - state_store(), - ingestion, - ); - assert_eq!(outputs.len(), 1); - let store = InMemoryDataStore::default() - .with_materialized_view(stored as usize, outputs) - .unwrap(); - run(&compile(&dag, &views[1..], &root).unwrap(), store, query) -} - -/// Rate evaluations and grouped Sum can run together at ingestion into a -/// stored Sum state, which answers as the query-time Rate→Sum over the stored -/// Rate state. Moving Sum before Rate would answer differently across -/// counter resets. -#[test] -fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { - let dag = grouped_rate(); - let state = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate, _), - .. - }) - ) - }) - .unwrap(); - let input_schema = Arc::new(state.output_schema.clone()); - let (family, update, grouping) = match &state.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family, - input, - grouping, - .. - }) => (family, input, grouping), - _ => unreachable!(), - }; - let range_ms = Some((-58_000, 2_000)); - let mut expected_rate_sum = 0.; - let rows = [[100., 0., 100.], [100., 200., 0.]] - .into_iter() - .enumerate() - .map(|(index, values)| { - let mut accumulator = create_planner_accumulator(family, update, grouping).unwrap(); - for (i, value) in values.into_iter().enumerate() { - accumulator.update_single(value, i as i64 * 1000).unwrap(); - } - let state = accumulator.into_accumulator(); - expected_rate_sum += state - .as_any() - .downcast_ref::() - .unwrap() - .evaluation(ExactKind::Rate, range_ms, None) - .unwrap() - .unwrap(); - let summary = Value::Summary { - family: family.clone(), - state: Arc::from(state), - }; - input_schema - .fields - .iter() - .map(|field| match &field.dtype { - FieldDataType::ExactAggregate(..) => summary.clone(), - FieldDataType::Plain(DataType::Timestamp) => Value::Timestamp(2000), - FieldDataType::Plain(DataType::Utf8) => Value::Utf8(if field.name == "job" { - "api".into() - } else { - serde_json::to_string(&BTreeMap::from([ - ("__name__", "m".to_string()), - ("job", "api".to_string()), - ("instance", format!("series-{index}")), - ])) - .unwrap() - .into() - }), - _ => panic!("unexpected input field {field:?}"), - }) - .collect() - }) - .collect(); - let batch = Batch::try_new(input_schema.clone(), rows).unwrap(); - let state_id = state.id as u64; - let sum_id = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }) - ) - }) - .unwrap() - .id as u64; - let ingestion = Scope::Ingestion { - window_start_ms: -58_000, - window_end_ms: 2000, - }; - let query = Scope::Query { - evaluation_time_ms: 2000, - }; - let mut results = vec![]; - for stored in [None, Some(sum_id)] { - let output = answer( - &dag, - state_id, - &batch, - stored, - ingestion.clone(), - query.clone(), - ); - assert_eq!(output.len(), 1, "{stored:?}"); - assert_eq!(output[0].to_rows().len(), 1); - assert!(matches!(&output[0].to_rows()[0][0], Value::Utf8(job) if job.as_ref() == "api")); - assert!( - matches!(output[0].to_rows()[0][1], Value::Float64(value) if value == expected_rate_sum) - ); - results.push( - output[0].to_rows()[0] - .iter() - .map(|value| value.key().unwrap()) - .collect::>(), - ); - } - assert_eq!(results[0], results[1]); - let mut wrong_order = create_planner_accumulator(family, update, grouping).unwrap(); - for (i, value) in [200., 200., 100.].into_iter().enumerate() { - wrong_order.update_single(value, i as i64 * 1000).unwrap(); - } - let rate_of_sum = wrong_order - .into_accumulator() - .as_any() - .downcast_ref::() - .unwrap() - .evaluation(ExactKind::Rate, range_ms, None) - .unwrap() - .unwrap(); - assert_ne!( - expected_rate_sum, rate_of_sum, - "counter resets prohibit moving Sum before Rate" - ); -} - -/// Every Pass 1 computation candidate of the grouped Rate answers a numeric -/// value at query time, from the stored Rate state or a stored grouped Sum. -#[test] -fn enumerated_grouped_rate_candidates_execute_numeric_query_outputs() { - let mut executed = 0; - for root in computation_candidates(grouped_rate_root()) { - let dag = compile_physical_asap_dag(&root).unwrap(); - let Some(state) = dag.nodes.iter().find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate, _), - .. - }) - ) - }) else { - continue; - }; - let boundary = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }) - ) - }) - .map(|node| node.id as u64) - .unwrap_or(dag.roots[0] as u64); - let (family, input, grouping) = match &state.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family, - input, - grouping, - .. - }) => (family, input, grouping), - _ => unreachable!(), - }; - let schema = Arc::new(state.output_schema.clone()); - let rows = ["a", "b"] - .into_iter() - .map(|instance| { - let mut accumulator = create_planner_accumulator(family, input, grouping).unwrap(); - for (timestamp, value) in [(1_000, 1.), (31_000, 31.), (59_000, 59.)] { - accumulator.update_single(value, timestamp).unwrap(); - } - let summary = Value::Summary { - family: family.clone(), - state: Arc::from(accumulator.into_accumulator()), - }; - schema - .fields - .iter() - .map(|field| match &field.dtype { - FieldDataType::ExactAggregate(..) => summary.clone(), - FieldDataType::Plain(DataType::Timestamp) => Value::Timestamp(60_000), - FieldDataType::Plain(DataType::Utf8) - if field.name == "$promql_series_identity" => - { - Value::Utf8( - serde_json::to_string(&BTreeMap::from([ - ("job", "api"), - ("instance", instance), - ])) - .unwrap() - .into(), - ) - } - FieldDataType::Plain(DataType::Utf8) => Value::Utf8("api".into()), - _ => panic!("unexpected input field {field:?}"), - }) - .collect() - }) - .collect(); - let batch = Batch::try_new(schema, rows).unwrap(); - for stored in [None, Some(boundary)] { - let output = answer( - &dag, - state.id as u64, - &batch, - stored, - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 60_000, - }, - Scope::Query { - evaluation_time_ms: 60_000, - }, - ); - assert_eq!(output.len(), 1); - assert_eq!(output[0].to_rows().len(), 1); - assert!(output[0] - .schema() - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); - assert!( - output[0].to_rows()[0] - .iter() - .any(|value| matches!(value, Value::Float64(x) if (*x - 2.).abs() < 1e-12)), - "{:?}", - output[0].to_rows() - ); - executed += 1; - } - } - assert!( - executed >= 2, - "must execute both stored and query-time grouped Rate candidates: {executed}" - ); -} diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 81e9b58cb..2ad42fbf7 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -7,14 +7,15 @@ use asap_executor::{ }; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::{GroupKeys, Reduction}; -use planner_types::ir::physical_export::{ - EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, - PhysicalASAPOperatorPayload, WindowEdgeCompatibility, -}; use planner_types::ir::properties::*; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{DataType, *}; use planner_types::ir::ASAPOp; +use planner_types::plan::physical_asap_dag::{ + EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, + PhysicalASAPOperatorPayload, WindowEdgeCompatibility, +}; +use planner_types::summary_type::*; use std::sync::Arc; fn logical_schema(family: FieldDataType) -> Schema { @@ -30,14 +31,10 @@ fn logical_schema(family: FieldDataType) -> Schema { time_index: None, } } -/// The stored state at node 0 (`MatAtIngestion`): a global `family` +/// The stored state at node 0 (materialized at ingestion time): a global `family` /// `SummaryAgg` over a minute of raw `m` samples (`Scan` node `scan`, /// `TimeRange` node `scan + 1`). -fn view_over_samples( - family: FieldDataType, - grouping: GroupingStrategy, - scan: usize, -) -> [PhysicalASAPDAGNode; 3] { +fn view_over_samples(family: FieldDataType, scan: usize) -> [PhysicalASAPDAGNode; 3] { let samples = Schema::with_time_index( vec![ Field::plain("time", DataType::Timestamp, false), @@ -50,23 +47,24 @@ fn view_over_samples( id, payload, output_state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; [ PhysicalASAPDAGNode { - materialization: planner_types::physical::Materialization::MatAtIngestion( - planner_types::physical::Medium::Memory, + lifetime: planner_types::stages::Lifetime::AcrossEvaluations( + planner_types::stages::StorageMedium::Memory, ), ..node( 0, PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::Reduce(GroupKeys::by(vec![])), - grouping, filter: None, child: scan + 1, }), @@ -125,24 +123,24 @@ fn try_state_dag( merge: bool, ) -> Result { let scan = 2 + usize::from(merge) + usize::from(target.is_some()); - let [view, scan_node, range_node] = - view_over_samples(family.clone(), GroupingStrategy::default(), scan); + let [view, scan_node, range_node] = view_over_samples(family.clone(), scan); let mut nodes = vec![view]; if merge { nodes.push(PhysicalASAPDAGNode { id: 1, payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryMerge { children: vec![0], - newest_tumbling_windows: None, group_by: None, }), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::physical::Materialization::MatAtIngestion( - planner_types::physical::Medium::Memory, + lifetime: planner_types::stages::Lifetime::AcrossEvaluations( + planner_types::stages::StorageMedium::Memory, ), + parameters: None, output_schema: logical_schema(family.clone()), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }); } let read_id = nodes.len(); @@ -152,29 +150,32 @@ fn try_state_dag( child: read_id - 1, }), output_state: ExecutionDataState::INGESTION_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: logical_schema(FieldDataType::Plain(DataType::Float64)), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }); if let Some(target) = target { nodes.push(PhysicalASAPDAGNode { id: nodes.len(), payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: target.clone(), + summary_type: target.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, child: read_id, }), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::physical::Materialization::MatAtIngestion( - planner_types::physical::Medium::Memory, + lifetime: planner_types::stages::Lifetime::AcrossEvaluations( + planner_types::stages::StorageMedium::Memory, ), + parameters: None, output_schema: logical_schema(target), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }); } let root = nodes.last().unwrap().id; @@ -234,8 +235,8 @@ fn ingestion_context(limits: Limits) -> RunContext { fn sum_state(value: f64) -> Arc { let mut state = asap_executor::summary_kernels::exact::ExactAccumulator::new( planner_types::ir::schema::FieldDataType::ExactAggregate( - planner_types::ir::schema::ExactKind::Sum, - planner_types::ir::schema::ExactParams::Sum, + planner_types::summary_type::ExactKind::Sum, + planner_types::summary_type::ExactParams::Sum, ), false, ) @@ -286,13 +287,7 @@ fn explicit_merge_changes_tumbling_window_cardinality() { #[test] fn precompute_rejects_nonfinite_and_out_of_range_dds_updates() { let source = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let target = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - GroupingStrategy::default(), - ); + let target = FieldDataType::Sketch(SketchAlgorithm::DDSketch); let program = state_dag(source.clone(), Some(target), false); for value in [20., -20., 0.] { assert!( diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 8bff8a966..99d260de4 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -8,15 +8,15 @@ use asap_executor::{ }; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::BinaryOpKind; -use planner_types::ir::physical_export::{ - EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, - PhysicalASAPOperatorPayload, WindowEdgeCompatibility, -}; use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::ArithmeticOpKind; use planner_types::ir::schema::{DataType, Field, FieldDataType}; use planner_types::ir::ASAPOp; use planner_types::ir::BinaryOperator; +use planner_types::plan::physical_asap_dag::{ + EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, + PhysicalASAPOperatorPayload, WindowEdgeCompatibility, +}; use std::{collections::BTreeMap, sync::Arc, time::Duration}; fn schema() -> SchemaRef { @@ -81,10 +81,12 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { rhs: 1, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*schema).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; compile_node(&node, &[schema.clone(), schema]).unwrap() } @@ -327,6 +329,7 @@ fn stored_series_evaluations_support_filters_and_sets() { use planner_types::ir::properties::*; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::*; + use planner_types::summary_type::*; use planner_types::ir::operator::PromQLVectorSetOpKind; use planner_types::ir::scalar::CompareOpKind; @@ -372,10 +375,9 @@ fn stored_series_evaluations_support_filters_and_sets() { payload: match id { 0 | 1 => PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 6, - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::Reduce(GroupKeys::by(vec![0])), - grouping: GroupingStrategy::PerSubpopulationInstance, filter: None, }), 5 => PhysicalASAPOperatorPayload::NonASAP( @@ -416,7 +418,8 @@ fn stored_series_evaluations_support_filters_and_sets() { 5 | 6 => ExecutionDataState::QUERY_ROWS, _ => ExecutionDataState::QUERY_ROWS, }, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: match id { 0 | 1 => (*state_schema).clone(), 5 | 6 => sample_schema(), @@ -424,6 +427,7 @@ fn stored_series_evaluations_support_filters_and_sets() { }, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }) .collect::>(); let edges = [ diff --git a/crates/executor/tests/promql_values.rs b/crates/executor/tests/promql_values.rs index 0585c4198..9e04d1975 100644 --- a/crates/executor/tests/promql_values.rs +++ b/crates/executor/tests/promql_values.rs @@ -144,7 +144,7 @@ fn compile_aggregate( let reduction = match intent { AggIntent::Sum { .. } => Reduction::Sum(1), AggIntent::Avg { .. } => Reduction::Avg(1), - AggIntent::Count { .. } => Reduction::Count, + AggIntent::Count => Reduction::Count, AggIntent::Min { .. } => Reduction::Min(1), AggIntent::Max { .. } => Reduction::Max(1), _ => return Err(Error::Invalid("unsupported vector aggregate".into())), @@ -204,7 +204,8 @@ fn compile_exact_evaluation( ) -> Result { use asap_executor::operators::SummaryEvaluation; use asap_executor::summary_kernels::exact::ExactEvaluation; - use planner_types::ir::schema::{ExactKind, FieldDataType}; + use planner_types::ir::schema::FieldDataType; + use planner_types::summary_type::ExactKind; let FieldDataType::ExactAggregate(statistic, _) = family else { return Err(Error::Invalid( "exact evaluation requires an exact family".into(), @@ -328,9 +329,7 @@ fn grouping_preserves_unenumerated_labels_and_empty_label_semantics() { equal_rows( run( compile_aggregate( - &AggIntent::Count { - accuracy: planner_types::ir::properties::AccuracyTarget::exact(), - }, + &AggIntent::Count, &GroupKeys::by(vec![ColumnRef::Named("job".into())]), ) .unwrap(), @@ -525,6 +524,7 @@ fn scalar_broadcast_rejects_colliding_result_labels_after_recovery() { fn exact_state_evaluations_recover_and_finalize_tumbling_windows() { use asap_executor::factory::create_planner_accumulator; use planner_types::ir::schema::*; + use planner_types::summary_type::*; use std::sync::Arc; for (kind, params, expected) in [ @@ -540,8 +540,8 @@ fn exact_state_evaluations_recover_and_finalize_tumbling_windows() { .map(|samples| { let mut state = create_planner_accumulator( &family, + None, &SummaryUpdate::column(ColumnRef::SampleValue), - &GroupingStrategy::PerSubpopulationInstance, ) .unwrap(); for sample in samples { @@ -576,6 +576,7 @@ fn exact_state_evaluations_recover_and_finalize_tumbling_windows() { fn recovered_exact_counter_uses_window_and_omits_insufficient_samples() { use asap_executor::factory::create_planner_accumulator; use planner_types::ir::schema::*; + use planner_types::summary_type::*; use std::sync::Arc; for (kind, params, expected) in [ @@ -588,8 +589,8 @@ fn recovered_exact_counter_uses_window_and_omits_insufficient_samples() { .map(|count| { let mut state = create_planner_accumulator( &family, + None, &SummaryUpdate::column(ColumnRef::SampleValue), - &GroupingStrategy::PerSubpopulationInstance, ) .unwrap(); state.update_single(100., -50_000).unwrap(); diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 5cdafc837..9edc47cf9 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -8,16 +8,17 @@ use asap_executor::{ Error, }; use futures::{executor::block_on, stream, StreamExt}; -use planner_types::ir::materialized_view::MaterializedView; use planner_types::ir::operator::{GroupKeys, Source}; -use planner_types::ir::physical_export::{ - EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, - PhysicalASAPOperatorPayload, WindowEdgeCompatibility, -}; use planner_types::ir::properties::*; use planner_types::ir::schema::{DataType, Field, *}; use planner_types::ir::NonASAPOp; use planner_types::ir::Predicate; +use planner_types::plan::materialized_view::MaterializedView; +use planner_types::plan::physical_asap_dag::{ + EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, + PhysicalASAPOperatorPayload, WindowEdgeCompatibility, +}; +use planner_types::summary_type::*; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -84,10 +85,12 @@ fn plan( id, payload, output_state: state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (**schema).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -504,10 +507,12 @@ fn windowed_plan(shift_ms: Option) -> PhysicalASAPDAG { id, payload: PhysicalASAPOperatorPayload::NonASAP(payload), output_state: state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -837,24 +842,25 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { id, payload, output_state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: output_schema.clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, } }; dag.nodes.push(node( 3, PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 2, - family, + summary_type: family, input: SummaryUpdate { item: None, weight: SummaryInputExpr::Constant(1.0), weight_domain: Default::default(), }, reduction: Reduction::Reduce(GroupKeys::by(vec![])), - grouping: GroupingStrategy::default(), filter: None, }), &state, diff --git a/crates/executor/tests/state_column_updater.rs b/crates/executor/tests/state_column_updater.rs index a87ee00bf..6569fcecc 100644 --- a/crates/executor/tests/state_column_updater.rs +++ b/crates/executor/tests/state_column_updater.rs @@ -1,13 +1,13 @@ //! The per-sample `StateColumnUpdater` builds the same states as the //! batch `SummaryAgg` build and as each family's kernel updated row by row, //! for every family a per-group build supports. -use asap_executor::factory::create_planner_accumulator; + use asap_executor::operators::{Operator, RowWeight}; use asap_executor::plan::PhysicalDAG; use asap_executor::planner::ir::scalar::ColumnRef; -use asap_executor::planner::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, HydraKind, - HydraParams, Schema, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, +use asap_executor::planner::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_executor::planner::summary_type::{ + ExactKind, ExactParams, SketchAlgorithm, SketchParams, SketchStatistic, }; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::summary_kernels::weighted_frequency::WeightedFrequency; @@ -91,63 +91,14 @@ fn input_batch() -> Batch { .unwrap() } -fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) +fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { + FieldDataType::Sketch(algorithm) } fn exact(kind: ExactKind, params: ExactParams) -> FieldDataType { FieldDataType::ExactAggregate(kind, params) } -/// Every family a per-group value build supports. -fn value_families() -> Vec { - vec![ - exact(ExactKind::Sum, ExactParams::Sum), - exact(ExactKind::Count, ExactParams::Count), - exact(ExactKind::Min, ExactParams::Min), - exact(ExactKind::Max, ExactParams::Max), - exact(ExactKind::Rate, ExactParams::Rate), - exact(ExactKind::Increase, ExactParams::Increase), - sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - // Above any group's sample count (at most 600): no - // compaction, so the random compaction coin never flips. - k: 1024, - failure_probability: None, - }, - ), - sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 10, - relative_error: None, - }, - ), - sketch( - SketchAlgorithm::Kmv, - SketchParams::Kmv { - k: 32, - relative_error: None, - }, - ), - sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 64, - sketch_rows: 5, - sketch_cols: 128, - layers: 4, - distinct_failure_probability: None, - }, - ), - ] -} - /// Every keyed family a per-group keyed build supports. fn keyed_families() -> Vec { vec![ @@ -201,7 +152,7 @@ fn fingerprint(state: &dyn AggregateCore) -> String { // Its heaps' internal order depends on hash-map iteration, so even // two merges of the same states differ in which heavy hitters they // keep, and so in their distinct count: compare its other answers. - use asap_executor::planner::ir::schema::SketchStatistic as Q; + use asap_executor::planner::summary_type::SketchStatistic as Q; let count = Q::PointCount { key: ColumnRef::SampleValue, value: None, @@ -281,223 +232,6 @@ fn batch_build(build: Operator) -> BTreeMap { /// One test sample: series, timestamp, value, item. type TestSample = (String, i64, f64, String); -/// How a test feeds one sample to an updater. -type FeedSample<'a> = dyn FnMut(&mut StateColumnUpdater, u32, &TestSample) + 'a; - -/// The per-sample build: one updater, groups numbered as series appear. -/// With `sharded`, the first half of the samples go to one updater and the -/// rest to another, merged at the end as a runtime's shards are. -fn per_sample_build( - family: &FieldDataType, - sharded: bool, - mut update: impl FnMut(&mut StateColumnUpdater, u32, &(String, i64, f64, String)), -) -> BTreeMap { - let samples = samples(); - let mut groups: BTreeMap = BTreeMap::new(); - for sample in &samples { - let next = groups.len() as u32; - groups.entry(sample.0.clone()).or_insert(next); - } - let shard = |part: &[(String, i64, f64, String)], update: &mut FeedSample<'_>| { - let mut updater = StateColumnUpdater::new(family).unwrap(); - updater.ensure_groups(groups.len()).unwrap(); - for sample in part { - update(&mut updater, groups[&sample.0], sample); - } - updater - }; - let split = if sharded { - samples.len() / 2 - } else { - samples.len() - }; - let (first, second) = samples.split_at(split); - let mut updater = shard(first, &mut update); - if sharded { - updater.merge(shard(second, &mut update)).unwrap(); - } - assert_eq!(updater.group_count(), groups.len()); - let states = updater.seal(); - groups - .into_iter() - .map(|(series, group)| (series, fingerprint(states.state(group as usize).as_ref()))) - .collect() -} - -/// The pre-columnar reference: each family's kernel, one per series, -/// updated row by row. With `sharded`, two halves merged through the -/// kernel's own merge. -fn kernel_build(family: &FieldDataType, keyed: bool, sharded: bool) -> BTreeMap { - let update = if keyed { - SummaryUpdate { - item: Some( - asap_executor::planner::ir::schema::SummaryInputExpr::Column(ColumnRef::Named( - "item".into(), - )), - ), - weight: asap_executor::planner::ir::schema::SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), - } - } else { - SummaryUpdate::column(ColumnRef::SampleValue) - }; - let samples = samples(); - let split = if sharded { - samples.len() / 2 - } else { - samples.len() - }; - let (first, second) = samples.split_at(split); - let half = |part: &[(String, i64, f64, String)]| { - let mut states = BTreeMap::new(); - for (series, ts, value, item) in part { - let state = states.entry(series.clone()).or_insert_with(|| { - if keyed { - let FieldDataType::Sketch(kind, _) = family else { - unreachable!() - }; - let (algorithm, width, depth, capacity) = match kind.params() { - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => ( - asap_executor::summary_kernels::weighted_frequency::FrequencyAlgorithm::Cms, - width, - depth, - heap_size, - ), - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => ( - asap_executor::summary_kernels::weighted_frequency::FrequencyAlgorithm::CountSketch, - width, - depth, - heap_size, - ), - _ => unreachable!(), - }; - Err(Box::new( - WeightedFrequency::new( - algorithm, - *width as usize, - *depth as usize, - *capacity as usize, - ) - .unwrap(), - )) - } else { - Ok(create_planner_accumulator(family, &update, &Default::default()).unwrap()) - } - }); - match state { - Ok(updater) => updater.update_value(&Value::Float64(*value), *ts).unwrap(), - Err(keyed) => keyed - .update(&[Value::Utf8(item.as_str().into())], 1.0) - .unwrap(), - } - } - states - .into_iter() - .map(|(series, state)| { - ( - series, - match state { - Ok(updater) => Arc::::from(updater.into_accumulator()), - Err(keyed) => Arc::new(*keyed) as Arc, - }, - ) - }) - .collect::>() - }; - let (mut merged, second) = (half(first), half(second)); - for (series, state) in second { - let combined = match merged.get(&series) { - Some(existing) => Arc::from(existing.merge_with(state.as_ref()).unwrap()), - None => state, - }; - merged.insert(series, combined); - } - merged - .into_iter() - .map(|(series, state)| (series, fingerprint(state.as_ref()))) - .collect() -} - -// For every value family: the per-sample updater, the batch `SummaryAgg` -// build and the family's own kernel build the same state for every group, -// byte for byte; two updater shards merged equal the kernel's merge of the -// same two halves. -#[test] -fn per_sample_updates_build_the_batch_states_for_every_value_family() { - for family in value_families() { - let batch = batch_build( - Operator::per_group_summary_build(input_schema(), family.clone(), 2, Some(1), vec![0]) - .unwrap(), - ); - let update = - |updater: &mut StateColumnUpdater, group, sample: &(String, i64, f64, String)| { - updater.update(group, sample.2, sample.1).unwrap() - }; - assert_eq!(batch.len(), 7, "{family:?}"); - assert_eq!( - per_sample_build(&family, false, update), - batch, - "{family:?}: updater vs batch build" - ); - assert_eq!( - kernel_build(&family, false, false), - batch, - "{family:?}: kernel vs batch build" - ); - assert_eq!( - per_sample_build(&family, true, update), - kernel_build(&family, false, true), - "{family:?}: merged shards vs kernel merge" - ); - } -} - -// The same for the keyed families (an item and a weight per sample). -#[test] -fn per_sample_updates_build_the_batch_states_for_every_keyed_family() { - for family in keyed_families() { - let batch = batch_build( - Operator::per_group_keyed_summary_build( - input_schema(), - family.clone(), - vec![3], - RowWeight::Constant(NonZeroU32::new(1).unwrap()), - vec![0], - ) - .unwrap(), - ); - let update = - |updater: &mut StateColumnUpdater, group, sample: &(String, i64, f64, String)| { - updater - .update_item(group, &[Value::Utf8(sample.3.as_str().into())], 1.0) - .unwrap() - }; - assert_eq!(batch.len(), 7, "{family:?}"); - assert_eq!( - per_sample_build(&family, false, update), - batch, - "{family:?}: updater vs batch build" - ); - assert_eq!( - kernel_build(&family, true, false), - batch, - "{family:?}: kernel vs batch build" - ); - assert_eq!( - per_sample_build(&family, true, update), - kernel_build(&family, true, true), - "{family:?}: merged shards vs kernel merge" - ); - } -} // A counter refuses a sample that is not after its group's previous one, // and a value outside the kernel's domain, leaving the state unchanged. @@ -593,24 +327,7 @@ fn an_unknown_group_is_bad_input() { /// A HydraCms family: one shared grid for all groups. fn hydra_family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 64, - depth: 3, - }, - ), - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: 64, - depth: 3, - shared_rows: 3, - shared_columns: 16, - }, - }, - ) + FieldDataType::Sketch(SketchAlgorithm::HydraCms) } /// The per-sample HydraCms build: groups added with their key as series diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index 350e66212..cc8019aa3 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -5,9 +5,8 @@ use asap_executor::arrow::array::{DictionaryArray, UInt32Array}; use asap_executor::arrow::datatypes::UInt32Type; use asap_executor::operators::{Operator, SortKey}; use asap_executor::plan::PhysicalDAG; -use asap_executor::planner::ir::schema::{ - DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchKind, SketchParams, -}; +use asap_executor::planner::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_executor::planner::summary_type::SketchAlgorithm; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, BatchColumn}; use asap_executor::{Error, StateColumnUpdater}; @@ -27,16 +26,7 @@ fn tumbling_window() -> (Arc, Batch) { /// One tumbling window of `series` KLL states (k = 200), keyed by series. fn tumbling_window_of(series_count: u32) -> (Arc, Batch) { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let schema = Arc::new(Schema { fields: vec![ Field { @@ -287,7 +277,7 @@ impl asap_executor::sources::DataStore for StoredTumblingWindows { } fn read_materialized_view( &self, - _: &asap_executor::planner::ir::materialized_view::MaterializedView, + _: &asap_executor::planner::plan::materialized_view::MaterializedView, _: &asap_executor::sources::MaterializedViewRequest, _: RunContext, ) -> Result, Error> { @@ -304,16 +294,16 @@ impl asap_executor::sources::DataStore for StoredTumblingWindows { /// `latency` (`Scan` node 2, `TimeRange` node 3). fn merge_over_a_stored_tumbling_window( schema: &Schema, -) -> asap_executor::planner::ir::physical_export::PhysicalASAPDAG { +) -> asap_executor::planner::plan::physical_asap_dag::PhysicalASAPDAG { use asap_executor::planner::ir::operator::{GroupKeys, Reduction, Source}; - use asap_executor::planner::ir::physical_export::{ - EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, - PhysicalASAPDAGNode, PhysicalASAPOperatorPayload as Payload, WindowEdgeCompatibility, - }; use asap_executor::planner::ir::properties::ExecutionDataState; use asap_executor::planner::ir::scalar::ColumnRef; - use asap_executor::planner::ir::schema::{GroupingStrategy, SummaryUpdate}; use asap_executor::planner::ir::{ASAPOp, NonASAPOp, TimeRangeKind}; + use asap_executor::planner::plan::physical_asap_dag::{ + EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, + PhysicalASAPDAGNode, PhysicalASAPOperatorPayload as Payload, WindowEdgeCompatibility, + }; + use asap_executor::planner::summary_type::SummaryUpdate; let samples = Schema::with_time_index( vec![ Field::plain("series", DataType::Utf8, false), @@ -328,19 +318,20 @@ fn merge_over_a_stored_tumbling_window( payload, output_schema: output_schema.clone(), output_state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let nodes = vec![ node( 0, Payload::ASAP(ASAPOp::SummaryAgg { child: 3, - family: schema.fields[1].dtype.clone(), + summary_type: schema.fields[1].dtype.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::Reduce(GroupKeys::by(vec![0])), - grouping: GroupingStrategy::default(), filter: None, }), schema, @@ -350,7 +341,6 @@ fn merge_over_a_stored_tumbling_window( 1, Payload::ASAP(ASAPOp::SummaryMerge { children: vec![0], - newest_tumbling_windows: None, group_by: None, }), schema, diff --git a/crates/executor/tests/summary_build_memory_limit.rs b/crates/executor/tests/summary_build_memory_limit.rs index 80af9b222..146e067ba 100644 --- a/crates/executor/tests/summary_build_memory_limit.rs +++ b/crates/executor/tests/summary_build_memory_limit.rs @@ -3,9 +3,8 @@ //! adds them, a bounded number of rows at a time. use asap_executor::operators::Operator; use asap_executor::plan::PhysicalDAG; -use asap_executor::planner::ir::schema::{ - DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchKind, SketchParams, -}; +use asap_executor::planner::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_executor::planner::summary_type::SketchAlgorithm; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, Value}; use asap_executor::Error; @@ -57,16 +56,7 @@ fn one_large_batch_fails_near_the_memory_limit() { .map(|i| vec![Value::Utf8(format!("s{i}").into()), Value::Float64(1.0)]) .collect(); let batch = Batch::try_new(input.clone(), rows).unwrap(); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let mut dag = PhysicalDAG::default(); dag.add( 0, diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index 392568d84..3e14c8b2e 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -9,15 +9,16 @@ use asap_executor::{ values::{Batch, Value}, }; use futures::{executor::block_on, StreamExt}; -use planner_types::ir::physical_export::{ - EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, - PhysicalASAPOperatorPayload, WindowEdgeCompatibility, -}; use planner_types::ir::properties::*; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{DataType, *}; use planner_types::ir::NonASAPOp; use planner_types::ir::{ASAPOp, TimeRangeKind}; +use planner_types::plan::physical_asap_dag::{ + EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, + PhysicalASAPOperatorPayload, WindowEdgeCompatibility, +}; +use planner_types::summary_type::*; use std::{sync::Arc, time::Duration}; /// The raw samples a stored summary is built from: `ts`, `value` and @@ -52,9 +53,11 @@ fn raw_node(id: usize, payload: NonASAPOp) -> PhysicalASAPDAGNode { payload: PhysicalASAPOperatorPayload::NonASAP(payload), output_schema: sample_schema(), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, } } @@ -113,19 +116,20 @@ fn summary_projection_survives_recovery() { id: 0, payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 3, - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: planner_types::ir::operator::Reduction::Reduce( planner_types::ir::operator::GroupKeys::by(vec![2]), ), - grouping: GroupingStrategy::PerSubpopulationInstance, filter: None, }), output_schema: (*schema).clone(), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }, raw_node(2, scan()), raw_node( @@ -151,9 +155,11 @@ fn summary_projection_survives_recovery() { }), output_schema: output.clone(), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }, ], edges: vec![ @@ -191,8 +197,8 @@ fn summary_projection_survives_recovery() { assert_eq!(*program.output_schema(1).unwrap(), output); let mut accumulator = create_planner_accumulator( &family, + None, &SummaryUpdate::column(ColumnRef::SampleValue), - &GroupingStrategy::PerSubpopulationInstance, ) .unwrap(); accumulator.update_single(7., 1).unwrap(); @@ -254,14 +260,15 @@ fn summary_merge_without_inputs_is_rejected() { id: 0, payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryMerge { children: vec![], - newest_tumbling_windows: None, group_by: None, }), output_schema: schema, output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }], edges: vec![], roots: vec![0], diff --git a/crates/executor/tests/time_series_window.rs b/crates/executor/tests/time_series_window.rs index 792a2881b..a05a19649 100644 --- a/crates/executor/tests/time_series_window.rs +++ b/crates/executor/tests/time_series_window.rs @@ -8,13 +8,12 @@ use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::{Source, TimeShift}; -use planner_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPDAG}; use planner_types::ir::scalar::{CompareOpKind, ScalarValue}; use planner_types::ir::schema::{DataType, Field, Schema, PROMQL_SERIES_IDENTITY}; use planner_types::ir::{ - apply_materialization_timings, ExprSemantics, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, + ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, }; +use planner_types::plan::physical_asap_dag::PhysicalASAPDAG; use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; @@ -84,13 +83,7 @@ fn window_over_rows( child, })) .unwrap(); - let timed = apply_materialization_timings( - &range, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() + planner_types::test_support::physical_asap_dag_at_query_time(&range).unwrap() } /// `(instance, ts, value)` samples of `latency_ms` as a store's raw data. diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 5533e6c31..2ff8822dd 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -10,31 +10,20 @@ use asap_executor::{ }; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::{GroupKeys, Reduction}; -use planner_types::ir::physical_export::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{DataType, Schema}; -use planner_types::ir::schema::{ - Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, -}; +use planner_types::ir::schema::{Field, FieldDataType}; use planner_types::ir::ASAPOp; +use planner_types::plan::physical_asap_dag::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; +use planner_types::summary_type::{ + NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, + WeightDomain, +}; use std::sync::Arc; fn family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 64, - sketch_rows: 5, - sketch_cols: 128, - layers: 4, - distinct_failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::UnivMon) } fn context() -> RunContext { @@ -178,7 +167,7 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() id: 1, payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 0, - family: family(), + summary_type: family(), input: SummaryUpdate { item: Some(SummaryInputExpr::Column(ColumnRef::Named("src_ip".into()))), weight: SummaryInputExpr::Constant(1.0), @@ -187,14 +176,15 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() }, }, reduction: Reduction::Reduce(GroupKeys::by(vec![])), - grouping: GroupingStrategy::default(), filter: None, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*state).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let build = compile_node(&node, std::slice::from_ref(&input))?; let batch = Batch::try_new( diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs index eacadcca7..90750507c 100644 --- a/crates/executor/tests/weighted_topk_binding.rs +++ b/crates/executor/tests/weighted_topk_binding.rs @@ -1,255 +1,10 @@ //! Planner output binds directly to the shared runtime over raw counter samples. -use crate::common; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::{ - physical_planner::{ - compile, - promql_rows::{series_row, with_series_identity}, - }, - runtime::{Limits, RunContext, Scope}, - values::{Batch, Value}, -}; -use common::{compile_physical_asap_dag, computation_candidates}; -use futures::{executor::block_on, StreamExt}; -use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; -use planner_types::ir::properties::AccuracyTarget; use planner_types::ir::schema::*; -use planner_types::ir::ASAPOp; -use std::{collections::BTreeMap, rc::Rc, sync::Arc}; -// The heap candidates of `topk by(job)(2, sum by(service, job)(rate(m[1m])))` -// rank the summed per-series rates of raw counter samples bound at the -// `Scan`, both maintained at ingestion and run at query time. -#[test] -fn planner_weighted_topk_binds_at_either_deployment_phase() { - assert_weighted_binding(SketchAlgorithm::CmsWithHeap); - assert_weighted_binding(SketchAlgorithm::CountSketchWithHeap); -} - -/// Whether `dag` builds a heap sketch of `algorithm`. -fn builds(dag: &PhysicalASAPDAG, algorithm: &SketchAlgorithm) -> bool { - dag.nodes.iter().any(|node| { - matches!(&node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if kind.algorithm() == algorithm) - }) -} - -/// The logical candidate of `root` at ε = 0.1 that builds a heap sketch of -/// `algorithm`. -fn heap_candidate( - root: Rc, - algorithm: &SketchAlgorithm, -) -> Rc { - computation_candidates(root) - .into_iter() - .find(|candidate| builds(&compile_physical_asap_dag(candidate).unwrap(), algorithm)) - .unwrap_or_else(|| panic!("missing {algorithm:?} candidate")) -} +use planner_types::summary_type::*; -/// `plan` as Stage 3 hands it to the executor when the data declares `m` a -/// counter: its rate weights' open requirement confirmed non-negative. -fn confirmed_as_counter( - plan: Rc, -) -> Rc { - use planner_types::ir::QueryRoot; - use planner_types::workload::{PrometheusMetricType, SourceFacts}; - let data = DataWorkload { - sources: [( - "m".to_string(), - SourceFacts { - metric_type: Some(PrometheusMetricType::Counter), - ..SourceFacts::default() - }, - )] - .into(), - ..DataWorkload::default() - }; - match planner_types::workload::bounds::with_confirmed_weights( - &[QueryRoot::Operator(plan)], - &data, - ) - .remove(0) - { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => unreachable!("an operator root"), - } -} - -fn assert_weighted_binding(algorithm: SketchAlgorithm) { - let root = lower_promql( - "topk by(job)(2, sum by(service, job)(rate(m[1m])))", - AccuracyTarget::max_error(0.1), - ) - .unwrap(); - // The per-series rate reads each raw series by its full identity. - let root = with_series_identity(&root).unwrap(); - let plan = confirmed_as_counter(heap_candidate(root, &algorithm)); - let dag = compile_physical_asap_dag(&plan).unwrap(); - let scan = dag - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::Scan { - source, - .. - }) => Some((source.clone(), Arc::new(node.output_schema.clone()))), - _ => None, - }) - .unwrap(); - let (source, samples) = scan; - // One counter series per rate: two samples 32 s apart in `(0, 60 s]`, - // the first an increase above zero, so `rate(m[1m])` extrapolates to the - // whole minute and is exactly `increase / 32 s`. The two `auth` series - // differ in `instance`. - let rows = [ - ("auth", "api", "a", 0.125), - ("auth", "api", "b", 0.25), - ("checkout", "api", "a", 0.3125), - ("search", "api", "a", 0.0625), - ("ingest", "batch", "a", 100.), - ("export", "batch", "a", 80.), - ("cleanup", "batch", "a", 20.), - ] - .into_iter() - .flat_map(|(service, job, instance, rate)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ("instance".to_string(), instance.to_string()), - ("job".to_string(), job.to_string()), - ("service".to_string(), service.to_string()), - ]); - let increase = rate * 32.; - [(14_000, increase), (46_000, 2. * increase)] - .map(|(ts, value)| series_row(&samples, &labels, ts, value).unwrap()) - }) - .collect(); - let batch = Batch::try_new(samples.clone(), rows).unwrap(); - // The summaries `MatAtIngestion` run as data arrives; `NotMaterialized` - // (the default) runs everything when the query does. - for (placed, scope) in [ - ( - continuously_maintained_dag(&plan), - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 60_000, - }, - ), - ( - dag.clone(), - Scope::Query { - evaluation_time_ms: 60_000, - }, - ), - ] { - let compiled = compile(&placed, &[], &[dag.roots[0] as u64]).unwrap(); - let store = InMemoryDataStore::default() - .with_raw_data(source.clone(), samples.clone(), vec![batch.clone()]) - .unwrap(); - let physical_dag = compiled.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new(scope, Limits::default()).unwrap(); - let output = block_on(async { - let mut output = Vec::new(); - let mut stream = physical_dag - .execute(&[dag.roots[0] as u64], context) - .unwrap() - .remove(0); - while let Some(batch) = stream.next().await { - output.extend(batch.unwrap().to_rows().iter().cloned()); - } - output - }); - // Two items per `job`: the outer `by(job)` partitions the ranking. - // Each row is `[job, topk_2, service]`: the top-k's columns, then - // the ranked item's label. - let schema = &dag - .nodes - .iter() - .find(|node| node.id == dag.roots[0]) - .unwrap() - .output_schema; - let column = |name: &str| schema.fields.iter().position(|f| f.name == name).unwrap(); - let (job, value) = (column("job"), column("topk_2")); - assert_eq!(column("service"), 2); - let mut ranked = output - .iter() - .map(|row| match (&row[job], &row[value]) { - (Value::Utf8(job), Value::Float64(score)) => (job.to_string(), *score), - other => panic!("unexpected (job, value) {other:?}"), - }) - .collect::>(); - ranked.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.total_cmp(&b.1))); - let expected = [ - ("api", 0.3125), - ("api", 0.375), - ("batch", 80.), - ("batch", 100.), - ]; - assert_eq!( - ranked, - expected.map(|(job, score)| (job.to_string(), score)), - "{}", - if placed == dag { - "query time" - } else { - "ingestion time" - } - ); - } -} - -use asap_frontend_promql::lower_promql_query_workload; -use planner_types::workload::{ - BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, -}; -pub fn lower_promql( - query: &str, - accuracy: AccuracyTarget, -) -> Result, asap_frontend_promql::PromqlError> { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy, - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1_000), - )), - }; - Ok(lower_promql_query_workload(&workload)? - .remove(0) - .as_operator() - .cloned() - .expect("operator root")) -} - -// The old untyped heap updater must not silently round a Planner rate update. #[test] fn rate_updates_cannot_enter_integer_heap_factory() { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 272, - depth: 5, - heap_size: 100, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); let input = SummaryUpdate { item: Some(SummaryInputExpr::Column( planner_types::ir::scalar::ColumnRef::Named("service".into()), @@ -259,45 +14,5 @@ fn rate_updates_cannot_enter_integer_heap_factory() { proof: NonNegativeWeightProof::ResetAwareCounterDerivative, }, }; - assert!(asap_executor::factory::create_planner_accumulator( - &family, - &input, - &Default::default() - ) - .is_err()); -} - -/// `candidate` with every `SummaryAgg` `MatAtIngestion`, the materialization -/// a continuously served query would get: each summary and every input it -/// consumes run at ingestion time, the rest at query time. The candidate pins -/// its finalize boundary to query time, which would put the summaries above -/// it upstream of a query-time node, so the pin is lifted first. -fn continuously_maintained_dag(candidate: &Rc) -> PhysicalASAPDAG { - use planner_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - Operator as IrOperator, OperatorNode, - }; - use planner_types::physical::{Materialization, Medium}; - fn unpinned(node: &Rc) -> Rc { - let mut copy = (**node).clone(); - copy.operator = node.operator.map_children(unpinned); - if matches!( - copy.operator, - IrOperator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) - ) { - copy.timing = None; - } - Rc::new(copy) - } - let candidate = unpinned(candidate); - let mut assignment = MaterializationAssignment::default(); - for node in OperatorNode::reachable(&candidate) { - if matches!(node.operator, IrOperator::ASAP(ASAPOp::SummaryAgg { .. })) { - assignment.set(&node, Materialization::MatAtIngestion(Medium::Memory)); - } - } - let timed = - apply_materialization_timings(&candidate, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - planner_types::ir::physical_export::compile_physical_asap_dag(&timed).unwrap() + assert!(asap_executor::factory::create_planner_accumulator(&family, None, &input).is_err()); } diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index 707acd89e..6123568cb 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -1,7 +1,7 @@ //! Post-lowering canonicalization of the operator DAG. //! //! Rewrites equivalent spellings of a query into one shape, so that common -//! sub-DAG sharing (Stage 0 hash-consing, `cse` in `asap_logical_asap_optimization`) can match +//! sub-DAG sharing (hash-consing, `Memo::find_or_add_group`) can match //! them by structural equality, whichever front end produced them. //! //! ## Heavy-hitter promotion @@ -38,12 +38,9 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; use asap_types::ir::operator::agg_intent::{topk, AggIntent}; +use asap_types::ir::operator::arguments::{GroupKeys, JoinKind, Reduction, WindowFuncKind}; use asap_types::ir::operator::node::{Operator, OperatorNode}; use asap_types::ir::operator::non_asap::NonASAPOp; -use asap_types::ir::operator::operator_properties::{ - GroupKeys, JoinKind, Reduction, WindowFuncKind, -}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use asap_types::ir::scalar::{ExprSemantics, Predicate, ProjectItem, ScalarExpr, SortKey}; use asap_types::ir::SchemaDerivationError; @@ -277,19 +274,12 @@ fn try_promote_additive_top_ranking( { return Ok(None); } - // Count ranks unit updates; a direct Sum ranks weighted updates. - let accuracy = match ranked_agg { - AggIntent::Count { accuracy } => accuracy.clone(), - AggIntent::Sum { .. } => AccuracyTarget::exact(), - _ => unreachable!("additive ranking gate admitted a non-additive measure"), - }; - // Outer heavy-hitter `TopK`, grouped by the ranking's partition (empty for // a global `ORDER BY … LIMIT k`; the `by` labels for a partitioned `topk // by`), over the unchanged inner additive aggregate. OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { reduction: Reduction::by(partition), - measures: vec![AggIntent::TopK { k: *k, accuracy }], + measures: vec![AggIntent::TopK { k: *k }], output_names: Vec::new(), filters: vec![], having: None, @@ -555,8 +545,8 @@ fn contains_scalar_subquery(expr: &ScalarExpr) -> bool { #[cfg(test)] mod tests { use super::*; - use asap_types::ir::operator::operator_properties::WindowFuncKind; - use asap_types::ir::operator::operator_properties::{ + use asap_types::ir::operator::arguments::WindowFuncKind; + use asap_types::ir::operator::arguments::{ GroupKeys, Source, WindowFrame, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, }; use asap_types::ir::schema::{DataType, Field, Schema}; @@ -597,9 +587,7 @@ mod tests { } fn count() -> AggIntent { - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - } + AggIntent::Count } /// `Aggregate{ by: [1], [Count] }` over the scan — output cols `[service, count]`. @@ -672,7 +660,7 @@ mod tests { return false; }; matches!(measures.as_slice(), [AggIntent::TopK { k: 5, .. }]) - && matches!(self::measures(child), [AggIntent::Count { .. }]) + && matches!(self::measures(child), [AggIntent::Count]) } #[test] diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index 1fb82f5a9..ea3d3b442 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -111,7 +111,7 @@ fn resolve( if child.schema.closed { return Ok(child); } - asap_types::ir::schema::schema_support::with_promql_series_identity(&child) + asap_types::ir::schema::series_identity::with_promql_series_identity(&child) .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e).into()) }; match tree { @@ -628,9 +628,7 @@ fn resolve_agg_intent( .transpose() }; Ok(match intent { - AggIntent::Count { accuracy } => AggIntent::Count { - accuracy: accuracy.clone(), - }, + AggIntent::Count => AggIntent::Count, AggIntent::PearsonCorr { left, right } => AggIntent::PearsonCorr { left: resolve_column_ref(left, schema)?, right: resolve_column_ref(right, schema)?, @@ -647,34 +645,19 @@ fn resolve_agg_intent( col: col(c)?, population: *population, }, - AggIntent::Quantile { - col: c, - q, - accuracy, - } => AggIntent::Quantile { + AggIntent::Quantile { col: c, q } => AggIntent::Quantile { col: col(c)?, q: *q, - accuracy: accuracy.clone(), - }, - AggIntent::TopK { k, accuracy } => AggIntent::TopK { - k: *k, - accuracy: accuracy.clone(), }, - AggIntent::Cardinality { cols, accuracy } => AggIntent::Cardinality { + AggIntent::TopK { k } => AggIntent::TopK { k: *k }, + AggIntent::Cardinality { cols } => AggIntent::Cardinality { cols: cols .iter() .map(|c| resolve_column_ref(c, schema)) .collect::>()?, - accuracy: accuracy.clone(), - }, - AggIntent::FrequencyL2 { col: c, accuracy } => AggIntent::FrequencyL2 { - col: col(c)?, - accuracy: accuracy.clone(), - }, - AggIntent::FrequencyEntropy { col: c, accuracy } => AggIntent::FrequencyEntropy { - col: col(c)?, - accuracy: accuracy.clone(), }, + AggIntent::FrequencyL2 { col: c } => AggIntent::FrequencyL2 { col: col(c)? }, + AggIntent::FrequencyEntropy { col: c } => AggIntent::FrequencyEntropy { col: col(c)? }, AggIntent::Rate => AggIntent::Rate, AggIntent::IRate => AggIntent::IRate, AggIntent::Increase => AggIntent::Increase, @@ -853,7 +836,7 @@ mod tests { use asap_types::ir::operator::{ BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Source, VectorMatch, }; - use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; use asap_types::ir::schema::{DataType, Field}; use asap_types::ir::BinaryOperator; @@ -936,18 +919,13 @@ mod tests { }; let intent = AggIntent::Cardinality { cols: vec![qualified("b"), qualified("a")], - accuracy: AccuracyTarget::exact(), }; assert_eq!( resolve_agg_intent(&intent, &schema).unwrap(), - AggIntent::Cardinality { - cols: vec![1, 0], - accuracy: AccuracyTarget::exact(), - } + AggIntent::Cardinality { cols: vec![1, 0] } ); let missing = AggIntent::Cardinality { cols: vec![qualified("a"), ColumnRef::Named("missing".into())], - accuracy: AccuracyTarget::exact(), }; assert!(resolve_agg_intent(&missing, &schema).is_err()); } diff --git a/crates/frontend-metricsql/src/lib.rs b/crates/frontend-metricsql/src/lib.rs index 4a3172506..23df96169 100644 --- a/crates/frontend-metricsql/src/lib.rs +++ b/crates/frontend-metricsql/src/lib.rs @@ -8,7 +8,6 @@ use asap_frontend_common::{ use asap_types::ir::operator::{ AggIntent, BinaryOpKind, GroupKeys, PromQLVectorSetOpKind, Reduction, Source, }; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::{BinaryOperator, ExprSemantics, TimeRangeKind}; use asap_types::workload::{DurationMs, PlanningWorkload, QueryLanguage, WorkloadError}; @@ -63,13 +62,7 @@ pub fn lower_metricsql_query_workload( workload .query_workload .entries() - .map(|entry| { - lower_with_sample_intervals( - &entry.query.0, - entry.requirements.accuracy.clone(), - &sample_interval_of, - ) - }) + .map(|entry| lower_with_sample_intervals(&entry.query.0, &sample_interval_of)) .collect() } @@ -79,7 +72,6 @@ pub fn lower_metricsql_query_workload( /// [`WorkloadError::MissingSampleInterval`]. pub(crate) fn lower_with_sample_intervals( query: &str, - accuracy: AccuracyTarget, sample_interval_of: &dyn Fn(&str) -> Option, ) -> Result { let ast = parse_metricsql(query)?; @@ -88,18 +80,13 @@ pub(crate) fn lower_with_sample_intervals( asap_types::ir::ScalarExpr::literal_f64(number.value), )); } - let unresolved = Lowerer { - accuracy, - sample_interval_of, - } - .lower(&ast)?; + let unresolved = Lowerer { sample_interval_of }.lower(&ast)?; resolve_root(&unresolved) .map(asap_types::ir::QueryRoot::Operator) .map_err(|e| MetricsqlError::Resolve(e.to_string())) } struct Lowerer<'a> { - accuracy: AccuracyTarget, /// What a bare instant selector over a metric looks back; `None` when /// the metric declares no sample interval. sample_interval_of: &'a dyn Fn(&str) -> Option, @@ -220,9 +207,7 @@ impl Lowerer<'_> { RollupFunction::MinOverTime => AggIntent::Min { col: None }, RollupFunction::MaxOverTime => AggIntent::Max { col: None }, RollupFunction::SumOverTime => AggIntent::Sum { col: None }, - RollupFunction::CountOverTime => AggIntent::Count { - accuracy: self.accuracy.clone(), - }, + RollupFunction::CountOverTime => AggIntent::Count, RollupFunction::StddevOverTime => AggIntent::StdDev { col: None, population: true, @@ -245,7 +230,6 @@ impl Lowerer<'_> { RollupFunction::QuantileOverTime => AggIntent::Quantile { col: None, q: number_arg(&function.args, 0)?, - accuracy: self.accuracy.clone(), }, _ => { return Err(unsupported(format!( @@ -294,9 +278,7 @@ impl Lowerer<'_> { AggregateFunction::Min => AggIntent::Min { col: None }, AggregateFunction::Max => AggIntent::Max { col: None }, // `count(v)` counts series, not distinct sample values. - AggregateFunction::Count => AggIntent::Count { - accuracy: self.accuracy.clone(), - }, + AggregateFunction::Count => AggIntent::Count, AggregateFunction::StdDev => AggIntent::StdDev { col: None, population: true, @@ -309,7 +291,6 @@ impl Lowerer<'_> { AggregateFunction::Quantile => AggIntent::Quantile { col: None, q: number_arg(&expr.args, 0)?, - accuracy: self.accuracy.clone(), }, _ => return Err(unsupported(format!("aggregate `{}`", expr.name()))), }; @@ -457,10 +438,9 @@ fn unsupported(message: impl Into) -> MetricsqlError { /// [`lower_metricsql_query`]). pub fn lower_metricsql( query: &str, - accuracy: AccuracyTarget, sample_interval: Duration, ) -> Result, MetricsqlError> { - match lower_metricsql_query(query, accuracy, sample_interval)? { + match lower_metricsql_query(query, sample_interval)? { asap_types::ir::QueryRoot::Operator(node) => Ok(node), _ => Err(MetricsqlError::UnsupportedFeature( "scalar root: use lower_metricsql_query".into(), @@ -473,8 +453,7 @@ pub fn lower_metricsql( /// back that long for its latest sample. pub fn lower_metricsql_query( query: &str, - accuracy: AccuracyTarget, sample_interval: Duration, ) -> Result { - lower_with_sample_intervals(query, accuracy, &|_| Some(sample_interval)) + lower_with_sample_intervals(query, &|_| Some(sample_interval)) } diff --git a/crates/frontend-metricsql/tests/external_corpus.rs b/crates/frontend-metricsql/tests/external_corpus.rs index 6aa8d1427..aafe36049 100644 --- a/crates/frontend-metricsql/tests/external_corpus.rs +++ b/crates/frontend-metricsql/tests/external_corpus.rs @@ -10,18 +10,6 @@ //! `ASAP_EXTERNAL_CORPUS_DIR= cargo test -p asap-frontend-metricsql //! --test metricsql external_corpus:: -- --nocapture` -use std::collections::HashMap; -use std::path::Path; -use std::time::Duration; - -use asap_frontend_metricsql::{lower_metricsql_query, MetricsqlError}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::QueryRoot; - -use crate::support::logical_plan; - -const DIR_VAR: &str = "ASAP_EXTERNAL_CORPUS_DIR"; - fn queries(corpus: &str) -> impl Iterator { corpus .lines() @@ -37,14 +25,6 @@ fn floor(corpus: &str) -> Option { .and_then(|n| n.trim().parse().ok()) } -#[derive(Default, Debug)] -struct Tally { - lowered: usize, - unparseable: usize, - rejected: usize, - bind_errored: usize, -} - /// `msg` with sanitized identifiers replaced by `_` and digit runs by `N`, so /// the same failure in different queries groups under one message. fn normalize(msg: &str) -> String { @@ -69,71 +49,6 @@ fn normalize(msg: &str) -> String { out } -/// Tally `corpus`, counting each failure's normalized message in `errors`. -fn tally(corpus: &str, errors: &mut HashMap) -> Tally { - let accuracy = AccuracyTarget::max_error(0.01); - let mut t = Tally::default(); - for q in queries(corpus) { - match lower_metricsql_query(q, accuracy.clone(), Duration::from_secs(15)) { - Ok(QueryRoot::Operator(dag)) => { - t.lowered += 1; - if logical_plan(&dag).is_err() { - t.bind_errored += 1; - } - } - Ok(QueryRoot::Scalar(_)) => t.lowered += 1, - Err(e) => { - if matches!(e, MetricsqlError::Parse(_)) { - t.unparseable += 1; - } else { - t.rejected += 1; - } - *errors.entry(normalize(&e.to_string())).or_default() += 1; - } - } - } - t -} - -/// Lowering and binding are total over every corpus file: a panic fails the -/// test, no lowerable query fails to bind, and coverage stays at its floor. -/// Also prints each unparseable/rejected cause with its count, most common -/// first, to show which unsupported features block the most queries. -#[test] -fn corpus_lowers_and_binds() { - let Some(dir) = std::env::var_os(DIR_VAR) else { - eprintln!("{DIR_VAR} unset; skipping the external corpus"); - return; - }; - let mut files: Vec<_> = std::fs::read_dir(Path::new(&dir)) - .unwrap_or_else(|e| panic!("{DIR_VAR}: {e}")) - .map(|entry| entry.unwrap().path()) - .filter(|path| path.extension().is_some_and(|ext| ext == "txt")) - .collect(); - files.sort(); - assert!(!files.is_empty(), "{DIR_VAR} has no *.txt files"); - - let mut failures = Vec::new(); - let mut errors = HashMap::new(); - for path in files { - let name = path.file_stem().unwrap().to_string_lossy(); - let corpus = std::fs::read_to_string(&path).unwrap(); - let min_lowered = floor(&corpus).unwrap_or_else(|| panic!("{name}: no `# floor: N` line")); - let t = tally(&corpus, &mut errors); - eprintln!("{name:24} {t:?}"); - if t.bind_errored > 0 || t.lowered < min_lowered { - failures.push(format!("{name}: {t:?} (floor {min_lowered})")); - } - } - let mut errors: Vec<_> = errors.into_iter().collect(); - errors.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); - eprintln!("\nFailures by cause:"); - for (msg, n) in errors { - eprintln!("{n:4} {msg}"); - } - assert!(failures.is_empty(), "{failures:#?}"); -} - #[test] fn floor_line() { assert_eq!(floor("# floor: 25\nrate(x[5m])"), Some(25)); diff --git a/crates/frontend-metricsql/tests/lowering.rs b/crates/frontend-metricsql/tests/lowering.rs index 92d200272..c8f972ebd 100644 --- a/crates/frontend-metricsql/tests/lowering.rs +++ b/crates/frontend-metricsql/tests/lowering.rs @@ -6,14 +6,13 @@ use asap_frontend_metricsql::test_support::{ }; use asap_frontend_metricsql::MetricsqlError; use asap_types::ir::operator::{AggIntent, Reduction, Source}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; /// Every metric's sample interval, the lookback of a bare selector. const INTERVAL: Duration = Duration::from_secs(15); fn lower(query: &str) -> Rc { - lower_metricsql(query, AccuracyTarget::max_error(0.01), INTERVAL).unwrap() + lower_metricsql(query, INTERVAL).unwrap() } #[test] @@ -71,12 +70,8 @@ fn default_rollup_with_explicit_range_is_last_over_time() { fn keep_metric_names_is_preserved_in_ast_and_rejected_without_lineage() { let ast = parse_metricsql("rate(requests_total[5m]) keep_metric_names").unwrap(); assert!(ast.keep_metric_names()); - let error = lower_metricsql( - "rate(requests_total[5m]) keep_metric_names", - AccuracyTarget::exact(), - INTERVAL, - ) - .unwrap_err(); + let error = + lower_metricsql("rate(requests_total[5m]) keep_metric_names", INTERVAL).unwrap_err(); assert!( matches!(error, MetricsqlError::UnsupportedFeature(message) if message.contains("metric-name lineage")) ); @@ -98,7 +93,7 @@ fn victoria_metrics_extension_corpus_parses_natively_and_fails_closed() { let ast = parse_metricsql(query) .unwrap_or_else(|error| panic!("native MetricsQL parser rejected {query:?}: {error}")); assert!(!ast.to_string().is_empty()); - let result = lower_metricsql(query, AccuracyTarget::exact(), INTERVAL); + let result = lower_metricsql(query, INTERVAL); assert!( result.is_err(), "extension semantics must be represented or routed to exact fallback: {query}" @@ -136,7 +131,7 @@ fn canonical_identity_recursively_formats_metricsql_extensions() { fn metricsql_multi_argument_aggregates_fail_closed() { for query in ["sum(foo, bar)", "avg(foo, bar)", "count(foo, bar)"] { parse_metricsql(query).expect("MetricsQL accepts multi-argument aggregates"); - let error = lower_metricsql(query, AccuracyTarget::exact(), INTERVAL).unwrap_err(); + let error = lower_metricsql(query, INTERVAL).unwrap_err(); assert!( matches!(error, MetricsqlError::UnsupportedFeature(message) if message.contains("requires exactly 1")), "{query} must not silently discard an aggregate input" diff --git a/crates/frontend-metricsql/tests/lowering_semantics.rs b/crates/frontend-metricsql/tests/lowering_semantics.rs index 3a1eb7842..37284b71b 100644 --- a/crates/frontend-metricsql/tests/lowering_semantics.rs +++ b/crates/frontend-metricsql/tests/lowering_semantics.rs @@ -3,7 +3,6 @@ use std::time::Duration; use asap_frontend_metricsql::{lower_metricsql_query_workload, MetricsqlError}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, DurationMs, PlanningWorkload, Query, QueryLanguage, @@ -12,11 +11,8 @@ use asap_types::workload::{ const INTERVAL: Duration = Duration::from_secs(15); -fn lower_metricsql( - query: &str, - accuracy: AccuracyTarget, -) -> Result, MetricsqlError> { - asap_frontend_metricsql::test_support::lower_metricsql(query, accuracy, INTERVAL) +fn lower_metricsql(query: &str) -> Result, MetricsqlError> { + asap_frontend_metricsql::test_support::lower_metricsql(query, INTERVAL) } fn workload(language: QueryLanguage, query: &str) -> PlanningWorkload { @@ -62,12 +58,12 @@ fn time_range_under_aggregate(node: &OperatorNode) -> (Duration, TimeRangeKind) /// `count(v)` counts series; it is not a distinct count of sample values. #[test] fn count_counts_series() { - let node = lower_metricsql("count(up)", AccuracyTarget::exact()).unwrap(); + let node = lower_metricsql("count(up)").unwrap(); let NonASAPOp::Aggregate { measures, .. } = node.expect_non_asap() else { panic!("expected an aggregate, got {node:?}"); }; assert!( - matches!(measures.as_slice(), [AggIntent::Count { .. }]), + matches!(measures.as_slice(), [AggIntent::Count]), "{measures:?}" ); } @@ -82,25 +78,25 @@ fn rollups_without_an_explicit_range_are_rejected() { "max_over_time(m)", "default_rollup(m)", ] { - let error = lower_metricsql(query, AccuracyTarget::exact()).unwrap_err(); + let error = lower_metricsql(query).unwrap_err(); assert!( matches!(&error, MetricsqlError::UnsupportedFeature(message) if message.contains("evaluation step")), "{query}: {error:?}" ); } - assert!(lower_metricsql("rate(m[5m])", AccuracyTarget::exact()).is_ok()); + assert!(lower_metricsql("rate(m[5m])").is_ok()); } /// A bare instant selector reads the latest sample within the sample /// interval, as in PromQL; an explicit range keeps its own window. #[test] fn instant_selectors_look_back_one_sample_interval() { - let node = lower_metricsql("sum(up)", AccuracyTarget::exact()).unwrap(); + let node = lower_metricsql("sum(up)").unwrap(); assert_eq!( time_range_under_aggregate(&node), (INTERVAL, TimeRangeKind::Instant) ); - let node = lower_metricsql("rate(m[5m])", AccuracyTarget::exact()).unwrap(); + let node = lower_metricsql("rate(m[5m])").unwrap(); assert_eq!( time_range_under_aggregate(&node), (Duration::from_secs(300), TimeRangeKind::Range) @@ -131,6 +127,7 @@ fn workload_lowering_uses_the_declared_sample_interval() { let mut other_metric = workload(QueryLanguage::MetricsQL, "sum(up) + sum(down)"); other_metric.data_workload.as_mut().unwrap().arrival = DataArrival::Ingesting { rows_per_second: 1.0, + max_out_of_order_ms: None, }; for workload in [missing, other_metric] { assert!(matches!( diff --git a/crates/frontend-promql/Cargo.toml b/crates/frontend-promql/Cargo.toml index b7bc12ad6..541f1f97f 100644 --- a/crates/frontend-promql/Cargo.toml +++ b/crates/frontend-promql/Cargo.toml @@ -39,10 +39,6 @@ path = "tests/observability/promql_corpus.rs" name = "o11y_bench_promql" path = "tests/observability/o11y_bench_promql.rs" -[[test]] -name = "metrics_observability" -path = "tests/observability/metrics_observability.rs" - [[test]] name = "promql" path = "tests/main.rs" diff --git a/crates/frontend-promql/src/lib.rs b/crates/frontend-promql/src/lib.rs index e6668bf66..fa90b507d 100644 --- a/crates/frontend-promql/src/lib.rs +++ b/crates/frontend-promql/src/lib.rs @@ -104,7 +104,6 @@ pub fn lower_promql_query_workload( .map(|entry| { let root = promql::PromqlLowerer::lower_query( &entry.query.0, - &entry.requirements.accuracy.clone(), &sample_interval_of, &metric_type_of, )?; @@ -252,6 +251,7 @@ mod tests { Some(DataArrival::Unknown), Some(DataArrival::Ingesting { rows_per_second: 1.0, + max_out_of_order_ms: None, }), None, ]; diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index 6c8a33f2a..8970cdff8 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -72,14 +72,13 @@ use asap_frontend_common::{ UnresolvedOp as Unresolved, UnresolvedPredicate, UnresolvedScalar as Scalar, UnresolvedSortKey, }; use asap_types::ir::operator::agg_intent::{topk, AggIntent, TimeFunc}; -use asap_types::ir::operator::operator_properties::{ +use asap_types::ir::operator::arguments::{ AtModifier, BinaryOpKind, GroupKeys, GroupSide, PromQLVectorSetOpKind, Reduction, Source, TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, }; use asap_types::ir::{BinaryOperator, ExprSemantics, TimeRangeKind}; use asap_types::ir::operator::{InfoMatcher, SampleKind}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::workload::{PrometheusMetricType, WorkloadError}; @@ -92,8 +91,6 @@ type Result = std::result::Result; /// Parses and lowers (→ the canonical, unresolved DAG) a PromQL query string. pub(crate) struct PromqlLowerer<'a> { - /// The target threaded onto every approximate intent. - accuracy: AccuracyTarget, /// What a bare instant selector over a metric looks back; `None` when /// the metric declares no sample interval. sample_interval_of: &'a dyn Fn(&str) -> Option, @@ -138,7 +135,6 @@ impl PromqlLowerer<'_> { /// checked by [`check_histogram_quantile_input`]. pub(crate) fn lower_query( query: &str, - accuracy: &AccuracyTarget, sample_interval_of: &dyn Fn(&str) -> Option, metric_type_of: &dyn Fn(&str) -> Option, ) -> Result { @@ -146,10 +142,7 @@ impl PromqlLowerer<'_> { check_depth(&ast, MAX_DEPTH)?; let mut metrics = Vec::new(); collect_metric_names(&ast, &mut metrics); - let lowerer = PromqlLowerer { - accuracy: accuracy.clone(), - sample_interval_of, - }; + let lowerer = PromqlLowerer { sample_interval_of }; if metrics .iter() .any(|metric| metric_type_of(metric) == Some(PrometheusMetricType::NativeHistogram)) @@ -402,7 +395,6 @@ impl PromqlLowerer<'_> { AggIntent::Quantile { col: None, q: num_arg(call, 0)?, - accuracy: self.accuracy.clone(), }, 1, ))), @@ -418,21 +410,10 @@ impl PromqlLowerer<'_> { col: None, population: true, }), - "count_over_time" => at0(AggIntent::Count { - accuracy: self.accuracy.clone(), - }), - "distinct_over_time" => at0(AggIntent::Cardinality { - cols: vec![], - accuracy: self.accuracy.clone(), - }), - "entropy_over_time" => at0(AggIntent::FrequencyEntropy { - col: None, - accuracy: self.accuracy.clone(), - }), - "l2_over_time" => at0(AggIntent::FrequencyL2 { - col: None, - accuracy: self.accuracy.clone(), - }), + "count_over_time" => at0(AggIntent::Count), + "distinct_over_time" => at0(AggIntent::Cardinality { cols: vec![] }), + "entropy_over_time" => at0(AggIntent::FrequencyEntropy { col: None }), + "l2_over_time" => at0(AggIntent::FrequencyL2 { col: None }), // Counter-derivative range functions (issue #44). Each has its own // intent — `changes` (value-change count) and `resets` (counter-reset // count) are NOT sample counts, so they are not aliased to @@ -536,9 +517,7 @@ impl PromqlLowerer<'_> { } } else if op == token::T_COUNT { // Counts vector elements regardless of their sample values. - Outer::Reduce(AggIntent::Count { - accuracy: self.accuracy.clone(), - }) + Outer::Reduce(AggIntent::Count) } else if op == token::T_SUM { Outer::Reduce(AggIntent::Sum { col: None }) } else if op == token::T_GROUP { @@ -581,7 +560,6 @@ impl PromqlLowerer<'_> { Outer::Reduce(AggIntent::Quantile { col: None, q: num_param(agg)?, - accuracy: self.accuracy.clone(), }) } else { return Err(LoweringError::UnsupportedAggregateOp(format!( @@ -632,7 +610,7 @@ impl PromqlLowerer<'_> { ) || (matches!(reduction, Reduction::PerEntity) && matches!( measures.as_slice(), - [AggIntent::Sum { .. } | AggIntent::Count { .. }] + [AggIntent::Sum { .. } | AggIntent::Count] ))) ); let direct_counter_ranking = matches!(&child, Unresolved::Aggregate { @@ -644,10 +622,7 @@ impl PromqlLowerer<'_> { if descending && (weighted_group_ranking || direct_counter_ranking) { return Ok(outer_aggregate( keys, - AggIntent::TopK { - k: k as usize, - accuracy: self.accuracy.clone(), - }, + AggIntent::TopK { k: k as usize }, child, )); } @@ -1089,7 +1064,7 @@ impl PromqlLowerer<'_> { // operator pair. The descending-plus-measure rule is shared with the // canonicalize-pass promotion so the two cannot drift (issue #38). let measure = match func { - Some(AggIntent::Count { .. }) => topk::Ranking::Frequency, + Some(AggIntent::Count) => topk::Ranking::Frequency, Some(AggIntent::Sum { .. }) => topk::Ranking::WeightedSum, _ => topk::Ranking::NonAdditive, }; @@ -1119,10 +1094,7 @@ impl PromqlLowerer<'_> { // A ranking always reduces (a `by`-empty TopK ranks the // whole input into one ordering, never per-entity). reduction: Reduction::Reduce(keys.into()), - measures: vec![AggIntent::TopK { - k: k as usize, - accuracy: self.accuracy.clone(), - }], + measures: vec![AggIntent::TopK { k: k as usize }], output_names: vec![], filters: vec![], having: None, diff --git a/crates/frontend-promql/tests/count_planning.rs b/crates/frontend-promql/tests/count_planning.rs index b208f7ae9..c53da52ea 100644 --- a/crates/frontend-promql/tests/count_planning.rs +++ b/crates/frontend-promql/tests/count_planning.rs @@ -1,42 +1,11 @@ //! Query text through summary selection: counts use observations, never value weights. use crate::planner_support::selected_dag; use crate::support; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{ - ExactKind, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SketchAlgorithm, - SummaryInputExpr, WeightDomain, -}; +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator, OperatorNode}; -use std::rc::Rc; -use support::{logical_candidates, lower_promql, physical_asap_dag}; - -/// Whether `node` reads a Hydra state (one sketch shared by every group). -fn reads_hydra(node: &Rc) -> bool { - OperatorNode::reachable(node).iter().any(|node| { - matches!( - &node.operator, - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { .. }), - .. - }) - ) - }) -} - -#[test] -#[ignore = "Pass 1 offers no Hydra state for a grouped count yet (#703)"] -fn grouped_count_offers_hydra_but_selection_keeps_per_group_state() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - // Hydra hashes a non-null item per row: the series identity (PromQL - // labels are nullable). - let root = asap_types::ir::schema::schema_support::with_promql_series_identity( - &lower_promql("count by(job)(up)", target.clone()).unwrap(), - ) - .unwrap(); - assert!(logical_candidates(Rc::clone(&root)).iter().any(reads_hydra)); - assert!(!reads_hydra(&selected_dag(root, target))); -} +use asap_types::summary_type::ExactKind; +use support::lower_promql; // Exact series and temporal counts must select a count accumulator, not distinct or sum. #[test] @@ -49,7 +18,7 @@ fn exact_counts_select_count_accumulators() { matches!( &node.operator, Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Count, _), + summary_type: FieldDataType::ExactAggregate(ExactKind::Count, _), .. }) ) @@ -59,82 +28,6 @@ fn exact_counts_select_count_accumulators() { } } -// CMS/UnivMon count updates stay +1 even for zero or negative samples. -// CountSketch cannot answer totals and must not be offered for count. -#[test] -fn frequency_count_candidates_use_unit_weights() { - for query in ["count_over_time(up[5m])", "count(up)"] { - let accuracy = AccuracyTarget::max_error(0.02); - let root = lower_promql(query, accuracy.clone()).unwrap(); - let candidates = logical_candidates(root); - let mut algorithms = Vec::new(); - for node in &candidates { - let Some(summary_input) = - OperatorNode::reachable(node) - .into_iter() - .find_map(|node| match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - Some(Rc::clone(summary_input)) - } - _ => None, - }) - else { - continue; - }; - let Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - input, - .. - }) = &summary_input.operator - else { - continue; - }; - assert!( - !matches!( - kind.algorithm(), - SketchAlgorithm::Hll | SketchAlgorithm::CountSketch - ), - "{query}: {node:?}" - ); - if !matches!( - kind.algorithm(), - SketchAlgorithm::Cms | SketchAlgorithm::UnivMon - ) { - continue; - } - let dag = physical_asap_dag(node); - assert!( - dag.nodes.iter().any(|node| matches!( - &node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { input: actual, .. }) if actual == input - )), - "post-ASAP DAG must preserve the count update contract" - ); - algorithms.push(*kind.algorithm()); - assert!(input.item.is_some(), "frequency keys must be explicit"); - assert_eq!( - input.weight, - SummaryInputExpr::Constant(1.0), - "{query}: {node:?}" - ); - assert_eq!( - input.weight_domain, - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount - } - ); - } - assert!( - algorithms.contains(&SketchAlgorithm::Cms), - "{query}: {candidates:?}" - ); - assert!( - algorithms.contains(&SketchAlgorithm::UnivMon), - "{query}: {candidates:?}" - ); - } -} - // Fixtures are already selected at one instant or within one five-minute window. // This narrow test oracle interprets the emitted aggregate, not Prometheus ingestion, // staleness, or scrape scheduling. Unsupported plan shapes fail explicitly. @@ -163,7 +56,7 @@ fn aggregate_fixture(query: &str, series: &[Vec]) -> Vec { other => panic!("unsupported fixture input: {other:?}"), } let aggregate = |values: &[f64]| match measures.as_slice() { - [AggIntent::Count { .. }] => values.len() as f64, + [AggIntent::Count] => values.len() as f64, [AggIntent::Cardinality { .. }] => { let mut distinct = values.to_vec(); distinct.sort_by(f64::total_cmp); @@ -225,57 +118,6 @@ fn count_over_time_counts_scrapes_not_sample_values() { ); } -// Execute the emitted CMS update-weight expression on the reported values. -// This checks the planner's numerical update contract, not a sketch-library runtime. -#[test] -fn cms_count_updates_total_ten_for_zero_positive_and_negative_samples() { - use asap_types::ir::scalar::ColumnRef; - let accuracy = AccuracyTarget::max_error(0.02); - let root = lower_promql("count_over_time(up[5m])", accuracy.clone()).unwrap(); - let candidates = logical_candidates(root); - let dag = candidates - .iter() - .find_map(|node| { - let dag = physical_asap_dag(node); - dag.nodes - .iter() - .any(|node| { - matches!(&node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::Cms) - }) - .then_some(dag) - }) - .expect("CMS count candidate"); - let update = dag - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { input, .. }) => Some(input), - _ => None, - }) - .unwrap(); - for value in [1.0, 0.0, 3.0, -3.0] { - let total: f64 = [value; 10] - .into_iter() - .map(|sample| { - let weight = match &update.weight { - SummaryInputExpr::Constant(value) => *value, - SummaryInputExpr::Column(ColumnRef::SampleValue) => sample, - SummaryInputExpr::Column(ColumnRef::Named(name)) if name == "value" => sample, - other => panic!("unsupported update weight: {other:?}"), - }; - assert!( - weight >= 0.0, - "CMS must not receive a negative update for {sample}" - ); - weight - }) - .sum(); - assert_eq!(total, 10.0, "ten samples of {value}"); - } -} - // The assertion helper must fail on an injected composition error; its coverage // must not depend on a real planner bug remaining unfixed. #[test] @@ -286,15 +128,3 @@ fn logical_candidates_fails_on_a_candidate_that_does_not_compose() { roots: Err("invalid choice".into()), }]); } - -// Ranked-row schemas let an aggregate consume TopK's numeric output (#672). -#[test] -fn aggregate_over_topk_candidates_compose() { - let accuracy = AccuracyTarget::max_error(0.01); - let root = lower_promql( - "sum by (job) (topk by (job, instance) (1, rate(m[5m])))", - accuracy.clone(), - ) - .unwrap(); - assert!(!logical_candidates(root).is_empty()); -} diff --git a/crates/frontend-promql/tests/histogram_metadata.rs b/crates/frontend-promql/tests/histogram_metadata.rs index 0968d8625..1fa25858f 100644 --- a/crates/frontend-promql/tests/histogram_metadata.rs +++ b/crates/frontend-promql/tests/histogram_metadata.rs @@ -9,8 +9,8 @@ //! series. use crate::support; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::workload::bounds::declared_non_negative; use asap_types::workload::{DurationMs, PrometheusMetricType, SourceFacts}; diff --git a/crates/frontend-promql/tests/main.rs b/crates/frontend-promql/tests/main.rs index 428e8a089..b962cb00b 100644 --- a/crates/frontend-promql/tests/main.rs +++ b/crates/frontend-promql/tests/main.rs @@ -12,4 +12,3 @@ mod promql_conformance; mod promql_equivalence; mod promql_lowering; mod scalar_design; -mod univmon_candidates; diff --git a/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs b/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs index 71bc59369..49b6a1039 100644 --- a/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs +++ b/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs @@ -31,8 +31,8 @@ use asap_frontend_promql::PromqlError as LoweringError; mod support; use std::rc::Rc; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::{AggIntent, BinaryOpKind, Reduction}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::CompareOpKind; use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode}; use support::lower_promql; diff --git a/crates/frontend-promql/tests/observability/metrics_observability.rs b/crates/frontend-promql/tests/observability/metrics_observability.rs deleted file mode 100644 index 39815ca2e..000000000 --- a/crates/frontend-promql/tests/observability/metrics_observability.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! Corpus totality checks over the metrics-observability benchmark exports. -//! -//! The source dump mixes Prometheus query logs, Grafana dashboard JSON, and -//! Prometheus rule YAML. The preprocessing script produces the line-oriented -//! fixtures consumed here. - -use asap_frontend_promql::PromqlError; -#[path = "../support.rs"] -mod support; -use asap_types::ir::properties::AccuracyTarget; -use support::{logical_plan, lower_promql}; - -/// Today's coverage of one corpus, as a ratchet: a change that plans fewer -/// queries with an ASAP operator, or fails to plan more of them, fails the -/// test. Raise `min_asap_candidates` (or lower `max_planning_errors`) when -/// coverage improves. -struct CoverageRatchet { - /// Queries whose logical plan binds an ASAP operator below the root. - min_asap_candidates: usize, - /// Lowered queries whose logical planning returns an error. - max_planning_errors: usize, -} - -const fn ratchet(min_asap_candidates: usize, max_planning_errors: usize) -> CoverageRatchet { - CoverageRatchet { - min_asap_candidates, - max_planning_errors, - } -} - -const CORPORA: &[(&str, &str, CoverageRatchet)] = &[ - ( - "CMU query logs", - include_str!("data/metrics_observability/cmu_chad_logs.txt"), - ratchet(162, 0), - ), - ( - "CMU dashboards", - include_str!("data/metrics_observability/cmu_chad_dashboards.txt"), - ratchet(174, 0), - ), - ( - "CMU rules", - include_str!("data/metrics_observability/cmu_chad_rules.txt"), - ratchet(202, 0), - ), - ( - "Claude query logs", - include_str!("data/metrics_observability/from_claude_logs.txt"), - ratchet(71, 0), - ), - ( - "Grafana dashboards", - include_str!("data/metrics_observability/grafana_dashboards.txt"), - ratchet(164, 0), - ), - ( - "Kubernetes mixin dashboards", - include_str!("data/metrics_observability/kubernetes_mixin_dashboards.txt"), - ratchet(147, 0), - ), - ( - "Kubernetes mixin alerts", - include_str!("data/metrics_observability/kubernetes_mixin_alerts.txt"), - ratchet(27, 2), - ), - ( - "Kubernetes mixin rules", - include_str!("data/metrics_observability/kubernetes_mixin_rules.txt"), - ratchet(66, 0), - ), - ( - "Promset", - include_str!("data/metrics_observability/promset.txt"), - ratchet(1505, 4), - ), -]; - -fn queries(corpus: &str) -> impl Iterator { - corpus - .lines() - .map(str::trim) - .filter(|line| !line.is_empty() && !line.starts_with('#')) -} - -#[test] -fn benchmark_corpora_are_total_and_keep_their_coverage() { - for (name, corpus, ratchet) in CORPORA { - let mut total = 0; - let mut lowered = 0; - let mut parse_failures = 0; - let mut rejected = 0; - let mut post_asap_candidates = 0; - let mut post_asap_unchanged = 0; - let mut post_asap_errors = 0; - - for query in queries(corpus) { - total += 1; - // Approximate accuracy exercises the sketch-replacement boundary - // used by the existing post-ASAP corpus binding test. - match lower_promql(query, AccuracyTarget::max_error(0.01)) { - Ok(expr) => { - lowered += 1; - match logical_plan(&expr) { - // An ASAP operator bound somewhere below the root. - Ok(node) if node.contains_asap() => post_asap_candidates += 1, - Ok(_) => { - post_asap_unchanged += 1; - if std::env::var_os("METRICS_OBSERVABILITY_REPORT").is_some() { - eprintln!("POST_ASAP_MISS\t{name}\t{query}"); - } - } - Err(error) => { - post_asap_errors += 1; - if std::env::var_os("METRICS_OBSERVABILITY_REPORT").is_some() { - eprintln!("POST_ASAP_ERROR\t{name}\t{query}\t{error}"); - } - } - } - } - Err(PromqlError::Parse(_)) => parse_failures += 1, - Err(_) => rejected += 1, - } - } - - eprintln!("{name}: total={total}, parse_errors={parse_failures}, lowering_errors={rejected}, pre_asap={lowered}, post_asap_candidates={post_asap_candidates}, post_asap_unchanged={post_asap_unchanged}, post_asap_errors={post_asap_errors}"); - assert!(total > 0, "{name} fixture is empty"); - // Dashboard variables and panel expressions can contain Grafana - // template syntax rather than standalone PromQL. They are still - // counted above so extraction drift is visible, but are not expected - // to parse as PromQL. - assert_eq!(total, lowered + rejected + parse_failures); - assert!( - post_asap_candidates >= ratchet.min_asap_candidates, - "{name}: {post_asap_candidates} queries bind an ASAP operator, fewer than {}", - ratchet.min_asap_candidates - ); - assert!( - post_asap_errors <= ratchet.max_planning_errors, - "{name}: {post_asap_errors} queries fail logical planning, more than {}", - ratchet.max_planning_errors - ); - } -} - -/// The two Kubernetes mixin alerts whose candidates do not compose: each -/// aggregates over a `topk` output, which the frontend declares as -/// `[keys.., topk_k]` with a `Utf8` placeholder and no value column, so the -/// outer `sum`'s accumulator has no value to read (and the plain plan would -/// sum a string). The ranked-row TopK contract now exposes its numeric value -/// and retained identity columns (#672). -#[test] -fn kubernetes_mixin_aggregates_over_topk_compose() { - for query in [ - r#"sum without (id, metrics_path, name, image, endpoint, job, node) ( topk by (cluster, namespace, pod, container, instance) (1, increase( container_cpu_cfs_throttled_periods_total{container!="", job="cadvisor", } [5m]) ) ) / on (cluster, namespace, pod, container, instance) group_left sum without (id, metrics_path, name, image, endpoint, job, node) ( topk by (cluster, namespace, pod, container, instance) (1, increase( container_cpu_cfs_periods_total{job="cadvisor", } [5m]) ) ) > ( 25 / 100 )"#, - r#"histogram_quantile(0.99, sum by (cluster, instance, le) ( topk by (cluster, instance, le, operation_type) (1, rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet"}[5m]) ) ) ) * on(cluster, instance) group_left(node) topk by (cluster, instance, node) (1, kubelet_node_name{job="kubelet"} ) > 60"#, - ] { - let root = lower_promql(query, AccuracyTarget::max_error(0.01)).unwrap(); - if let Err(error) = logical_plan(&root) { - panic!("{query}: {error}"); - } - } -} diff --git a/crates/frontend-promql/tests/observability/o11y_bench_promql.rs b/crates/frontend-promql/tests/observability/o11y_bench_promql.rs index 53d125bc1..ca8d21f5c 100644 --- a/crates/frontend-promql/tests/observability/o11y_bench_promql.rs +++ b/crates/frontend-promql/tests/observability/o11y_bench_promql.rs @@ -21,7 +21,7 @@ use asap_frontend_promql::PromqlError as LoweringError; #[path = "../support.rs"] mod support; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use support::lower_promql; const CORPUS: &str = include_str!("data/o11y_bench_promql.txt"); diff --git a/crates/frontend-promql/tests/observability/promql_corpus.rs b/crates/frontend-promql/tests/observability/promql_corpus.rs index cd0aab26c..b63d4b68f 100644 --- a/crates/frontend-promql/tests/observability/promql_corpus.rs +++ b/crates/frontend-promql/tests/observability/promql_corpus.rs @@ -16,8 +16,7 @@ use asap_frontend_promql::PromqlError as LoweringError; #[path = "../support.rs"] mod support; -use asap_types::ir::properties::AccuracyTarget; -use support::{logical_plan, lower_promql}; +use asap_types::accuracy::AccuracyTarget; const DOCS: &str = include_str!("data/promql_corpus_docs.txt"); const TESTDATA: &str = include_str!("data/promql_corpus_testdata.txt"); @@ -67,51 +66,6 @@ fn tally(corpus: &str) -> Tally { t } -/// Every query that lowers, additionally run through logical ASAP-aware -/// optimization ([`logical_plan`]: Pass 1 and composition), at an -/// approximate accuracy target so summary candidates actually appear (an -/// `Exact` target would only ever exercise the exact-accumulator arm). -#[derive(Default, Debug)] -struct BindTally { - /// An ASAP operator was bound somewhere below the root — the pass did - /// something. - transformed: usize, - /// The kept pre-ASAP dag — the pass left the query untouched. - unchanged: usize, - /// [`logical_plan`] returned `Err`: Pass 1 rejected the query, or its - /// exact candidate did not compose. - errored: usize, -} - -fn bind_tally(corpus: &str, accuracy: AccuracyTarget) -> BindTally { - let mut t = BindTally::default(); - for q in queries(corpus) { - let Ok(dag) = lower_promql(q, accuracy.clone()) else { - continue; - }; - match logical_plan(&dag) { - Ok(bound) if !bound.contains_asap() => t.unchanged += 1, - Ok(_) => t.transformed += 1, - Err(_) => t.errored += 1, - } - } - t -} - -#[test] -fn binding_is_total_over_the_entire_corpus() { - let accuracy = AccuracyTarget::max_error(0.01); - let docs = bind_tally(DOCS, accuracy.clone()); - let td = bind_tally(TESTDATA, accuracy); - eprintln!("docs corpus post-ASAP binding: {docs:?}"); - eprintln!("testdata corpus post-ASAP binding: {td:?}"); - - // Same totality guarantee as lowering: reaching here means `logical_plan` - // never panicked over any lowerable query in the corpus. - assert_eq!(docs.errored, 0, "post-ASAP binding errored: {docs:?}"); - assert_eq!(td.errored, 0, "post-ASAP binding errored: {td:?}"); -} - #[test] fn lowering_is_total_over_the_entire_corpus() { let docs = tally(DOCS); diff --git a/crates/frontend-promql/tests/planner_support.rs b/crates/frontend-promql/tests/planner_support.rs index 39f00e0ca..9be35e5f8 100644 --- a/crates/frontend-promql/tests/planner_support.rs +++ b/crates/frontend-promql/tests/planner_support.rs @@ -2,7 +2,7 @@ //! which the private corpus also compiles without depending on asap-planner. use std::rc::Rc; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::OperatorNode; use crate::support::declared_data; @@ -37,7 +37,8 @@ pub fn run_all_stages( /// at `accuracy`. pub fn selected_dag(root: Rc, accuracy: AccuracyTarget) -> Rc { let run = run_all_stages(root, accuracy); - match run.selected_plan.as_deref().expect("a plan is selected") { + let (plan, _) = run.selection.as_ref().expect("a plan is selected"); + match plan.query_roots().unwrap().as_slice() { [asap_types::ir::QueryRoot::Operator(node)] => Rc::clone(node), other => panic!("one operator root: {other:?}"), } diff --git a/crates/frontend-promql/tests/promql_binding_regressions.rs b/crates/frontend-promql/tests/promql_binding_regressions.rs index 77066e474..0d4e4784d 100644 --- a/crates/frontend-promql/tests/promql_binding_regressions.rs +++ b/crates/frontend-promql/tests/promql_binding_regressions.rs @@ -1,5 +1,5 @@ use crate::support; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use support::lower_promql; /// Prometheus treats these quantile parameters as valid queries returning special values. @@ -38,5 +38,5 @@ fn count_is_row_count_not_distinct_sample_value_count() { let NonASAPOp::Aggregate { measures, .. } = tree.expect_non_asap() else { panic!("expected aggregate") }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } diff --git a/crates/frontend-promql/tests/promql_conformance.rs b/crates/frontend-promql/tests/promql_conformance.rs index e661ee984..4ea487c36 100644 --- a/crates/frontend-promql/tests/promql_conformance.rs +++ b/crates/frontend-promql/tests/promql_conformance.rs @@ -37,11 +37,11 @@ use std::time::Duration; use crate::support; use asap_frontend_promql::PromqlError as LoweringError; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::{ AggIntent, AtModifier, BinaryOpKind, PromQLVectorSetOpKind, Reduction, SampleKind, Source, TimeFunc, }; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::schema::DataType; use asap_types::ir::{ @@ -289,10 +289,7 @@ fn sum_by_groups_via_positional_aggregate() { #[test] fn count_is_row_count() { - assert!(has(&ok("count(up)"), |i| matches!( - i, - AggIntent::Count { .. } - ))); + assert!(has(&ok("count(up)"), |i| matches!(i, AggIntent::Count))); } #[test] @@ -425,7 +422,7 @@ fn over_time_functions_reduce_over_time_range() { "max" => matches!(i, AggIntent::Max { .. }), "min" => matches!(i, AggIntent::Min { .. }), "sum" => matches!(i, AggIntent::Sum { .. }), - "count" => matches!(i, AggIntent::Count { .. }), + "count" => matches!(i, AggIntent::Count), _ => unreachable!(), }); assert!(matched, "{q}: missing {want} intent"); @@ -582,28 +579,14 @@ fn double_unary_negation_nests() { assert!(negates_via_scalar(child)); } +/// `count` lowers to a `Count` measure, not a distinct count. #[test] -fn count_maps_to_count_and_inherits_accuracy() { - // Counts preserve the workload accuracy target without counting distinct values. - let exact = lower_promql("count by (job) (up)", AccuracyTarget::exact()).unwrap(); - assert!( - has(&exact, |i| matches!( - i, - AggIntent::Count { accuracy } if accuracy.is_exact() - )), - "Count must stay exact under an exact target, got {:?}", - intents(&exact) - ); - - let approx = lower_promql("count by (job) (up)", AccuracyTarget::max_error(0.01)).unwrap(); +fn count_maps_to_count() { + let lowered = lower_promql("count by (job) (up)", AccuracyTarget::exact()).unwrap(); assert!( - has(&approx, |i| matches!( - i, - AggIntent::Count { accuracy } if (accuracy.max_error - 0.01).abs() < 1e-9 - && accuracy.accuracy_metric.is_none() - )), - "Count must carry the approximate target, got {:?}", - intents(&approx) + has(&lowered, |i| matches!(i, AggIntent::Count)), + "got {:?}", + intents(&lowered) ); } @@ -1939,7 +1922,7 @@ fn limitk_by_carries_the_grouping_and_composes_in_a_set_op() { // the PromqlSeriesSample must be preserved under the set op (it must lower, not reject). assert!(has( &ok("count(limitk(2, http_requests) and http_requests)"), - |i| matches!(i, AggIntent::Count { .. }) + |i| matches!(i, AggIntent::Count) )); } diff --git a/crates/frontend-promql/tests/promql_equivalence.rs b/crates/frontend-promql/tests/promql_equivalence.rs index 07ad8fa7d..d31e490d7 100644 --- a/crates/frontend-promql/tests/promql_equivalence.rs +++ b/crates/frontend-promql/tests/promql_equivalence.rs @@ -20,7 +20,7 @@ use std::rc::Rc; use crate::support; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::OperatorNode; use support::lower_promql; diff --git a/crates/frontend-promql/tests/promql_lowering.rs b/crates/frontend-promql/tests/promql_lowering.rs index 163fe82b6..0a92c4790 100644 --- a/crates/frontend-promql/tests/promql_lowering.rs +++ b/crates/frontend-promql/tests/promql_lowering.rs @@ -3,8 +3,8 @@ use std::rc::Rc; use std::time::Duration; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::{AggIntent, BinaryOpKind, Reduction, Source}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::{ BinaryOperator, ExprSemantics, NonASAPOp, OperatorNode, ScalarExpr, TimeRangeKind, @@ -30,17 +30,11 @@ fn frequency_extensions_lower_to_explicit_frequency_statistics() { for (query, expected) in [ ( "entropy_over_time(cpu_usage[5m])", - AggIntent::FrequencyEntropy { - col: None, - accuracy: AccuracyTarget::exact(), - }, + AggIntent::FrequencyEntropy { col: None }, ), ( "l2_over_time(cpu_usage[5m])", - AggIntent::FrequencyL2 { - col: None, - accuracy: AccuracyTarget::exact(), - }, + AggIntent::FrequencyL2 { col: None }, ), ] { assert!(all_intents(&lower(query)) @@ -50,9 +44,9 @@ fn frequency_extensions_lower_to_explicit_frequency_statistics() { } #[test] -fn distinct_over_time_preserves_cardinality_accuracy_and_nested_windows() { - // Distinct counts sample values, not samples or series; all lowering routes - // retain the caller's accuracy requirement, including subquery arguments. +fn distinct_over_time_lowers_to_cardinality_over_nested_windows() { + // Distinct counts sample values, not samples or series, on every lowering + // route, subquery arguments included. for query in [ "distinct_over_time(cpu_usage{job=\"worker\"}[5m] offset 1h)", "distinct_over_time((cpu_usage + 1)[5m:1m])", @@ -63,14 +57,14 @@ fn distinct_over_time_preserves_cardinality_accuracy_and_nested_windows() { let mut intents = Vec::new(); collect_intents(&dag, &mut intents); assert!( - intents.iter().any(|intent| matches!( - intent, AggIntent::Cardinality { accuracy: actual, .. } if *actual == accuracy - )), + intents + .iter() + .any(|intent| matches!(intent, AggIntent::Cardinality { .. })), "{query}: {dag:?}" ); assert!(!intents .iter() - .any(|intent| matches!(intent, AggIntent::Count { .. }))); + .any(|intent| matches!(intent, AggIntent::Count))); } } } @@ -441,7 +435,7 @@ fn count_over_rate_keeps_both_levels() { else { panic!("expected outer Aggregate{{Count}}, got {qe:?}"); }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); assert!(matches!( child.expect_non_asap(), NonASAPOp::Aggregate { measures, .. } if matches!(measures.as_slice(), [AggIntent::Rate]) @@ -473,7 +467,7 @@ fn count_over_distinct_over_time_preserves_both_aggregates() { panic!("expected outer Count: {dag:?}"); }; assert!( - matches!(measures.as_slice(), [AggIntent::Count { .. }]), + matches!(measures.as_slice(), [AggIntent::Count]), "{query}: {dag:?}" ); assert_eq!(actual, &reduction, "{query}"); @@ -513,7 +507,7 @@ fn count_never_lowers_to_distinct_sample_values() { let dag = lower(query); let intents = all_intents(&dag); assert!( - intents.iter().any(|i| matches!(i, AggIntent::Count { .. })), + intents.iter().any(|i| matches!(i, AggIntent::Count)), "{query}: {dag:?}" ); assert!( @@ -541,7 +535,7 @@ fn outer_count_counts_series() { panic!("expected outer Aggregate grouped by symbol, got {qe:?}"); }; assert_eq!(reduction, &Reduction::by(vec![2])); - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); // Inner: Aggregate{Count} over TimeRange (per-series count_over_time). let NonASAPOp::Aggregate { measures, child, .. @@ -549,7 +543,7 @@ fn outer_count_counts_series() { else { panic!("expected Aggregate (count_over_time) under the outer count, got {child:?}"); }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); assert!(matches!( child.expect_non_asap(), NonASAPOp::TimeRange { .. } @@ -584,7 +578,7 @@ fn topk_over_count_is_heavy_hitter_topk() { else { panic!("expected Aggregate (count_over_time) under TopK, got {child:?}"); }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); let NonASAPOp::TimeRange { range, child, .. } = child.expect_non_asap() else { panic!("expected TimeRange under Count aggregate, got {child:?}"); }; @@ -686,7 +680,7 @@ fn bottomk_over_count_is_generic_sort_ascending() { }; assert!(keys[0].ascending, "bottomk ranks ascending"); // Count intent is still present (as the inner aggregate), no TopK. - assert!(has_intent(&qe, |i| matches!(i, AggIntent::Count { .. }))); + assert!(has_intent(&qe, |i| matches!(i, AggIntent::Count))); assert!(!has_intent(&qe, |i| matches!(i, AggIntent::TopK { .. }))); } @@ -737,7 +731,7 @@ fn topk_count_output_schema_carries_group_key() { else { panic!("expected inner Aggregate{{Count}}, got {child:?}"); }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); assert!(matches!( child.expect_non_asap(), NonASAPOp::TimeRange { .. } @@ -1040,22 +1034,6 @@ fn fill_modifiers_are_rejected_not_ignored() { // ── accuracy propagation ────────────────────────────────────────────────────── -#[test] -fn accuracy_target_flows_into_quantile_intent() { - let qe = lower_promql( - "quantile_over_time(0.9, m[5m])", - AccuracyTarget::max_error(0.01), - ) - .unwrap(); - let NonASAPOp::Aggregate { measures, .. } = qe.expect_non_asap() else { - panic!("expected Aggregate"); - }; - assert!(matches!( - &measures[0], - AggIntent::Quantile { accuracy, .. } if (accuracy.max_error - 0.01).abs() < 1e-12 - )); -} - // ── schema flow (positional, carried on Scan; derived on demand) ───────────────── #[test] diff --git a/crates/frontend-promql/tests/scalar_design.rs b/crates/frontend-promql/tests/scalar_design.rs index 301e46c35..4111818fe 100644 --- a/crates/frontend-promql/tests/scalar_design.rs +++ b/crates/frontend-promql/tests/scalar_design.rs @@ -1,6 +1,6 @@ //! Scalar expressions never become constant-wrapper operators. use crate::support; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::{NonASAPOp, QueryRoot, ScalarExpr}; diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index 19834b75d..fa6b6db5f 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -1,7 +1,7 @@ use std::rc::Rc; use asap_frontend_promql::{lower_promql_query_workload, PromqlError}; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::scalar::ScalarValue; use asap_types::ir::{NonASAPOp, OperatorNode, ScalarExpr}; use asap_types::workload::{ @@ -76,24 +76,14 @@ pub fn promql_scalar(node: &ScalarExpr) -> Option { } } -/// Time `root` under the default materialization assignment (every summary +/// Time `root` under the default physical properties assignment (every summary /// at query time) and export the physical DAG — export needs every node /// timed first. #[allow(dead_code)] pub fn physical_asap_dag( root: &Rc, -) -> asap_types::ir::physical_export::PhysicalASAPDAG { - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - let timed = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .expect("default materialization timings"); - asap_types::ir::physical_export::compile_physical_asap_dag(&timed) - .expect("post-ASAP DAG export") +) -> asap_types::plan::physical_asap_dag::PhysicalASAPDAG { + asap_types::test_support::physical_asap_dag_at_query_time(root).expect("post-ASAP DAG export") } #[allow(dead_code)] @@ -141,50 +131,12 @@ pub fn declared_data(root: &Rc) -> DataWorkload { DataWorkload { arrival: asap_types::workload::DataArrival::Ingesting { rows_per_second: 1_000.0, + max_out_of_order_ms: None, }, sources, } } -/// Local computation candidates for frontend binding tests. Other computations -/// keep their default choices; a timed MILP archive is not a candidate catalog. -#[allow(dead_code)] -pub fn logical_candidates(root: Rc) -> Vec> { - local_computation_candidates(root).expect("generated computation candidates compose") -} - -fn local_computation_candidates(root: Rc) -> Result>, String> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::ir::QueryRoot; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .map_err(|error| error.to_string())?; - let mut choices = vec![vec![0; inventory.targets.len()]]; - for (index, target) in inventory.targets.iter().enumerate() { - for choice in 1..target.choice_count() { - let mut selected = vec![0; inventory.targets.len()]; - selected[index] = choice; - choices.push(selected); - } - } - choices - .into_iter() - .map(|choice| { - let mut roots = compose_logical_candidate(&inventory, &choice) - .map_err(|error| error.to_string())?; - match roots.remove(0) { - QueryRoot::Operator(root) => Ok(root), - QueryRoot::Scalar(_) => Err("expected an operator root".into()), - } - }) - .collect() -} - /// A composed logical candidate, or why it does not compose. #[allow(dead_code)] pub struct LogicalCandidate { @@ -211,15 +163,3 @@ pub fn require_composed_operator_candidates( }) .collect() } - -/// One generated ASAP candidate, or the exact computation if no local rule -/// applies. Corpus coverage tests exercise binding, not cost-based selection. -#[allow(dead_code)] -pub fn logical_plan(root: &Rc) -> Result, String> { - let candidates = local_computation_candidates(Rc::clone(root))?; - Ok(candidates - .iter() - .find(|node| node.contains_asap()) - .unwrap_or(&candidates[0]) - .clone()) -} diff --git a/crates/frontend-promql/tests/univmon_candidates.rs b/crates/frontend-promql/tests/univmon_candidates.rs deleted file mode 100644 index de94f5e2c..000000000 --- a/crates/frontend-promql/tests/univmon_candidates.rs +++ /dev/null @@ -1,153 +0,0 @@ -use std::rc::Rc; - -use crate::planner_support::run_all_stages; -use crate::support; -use asap_accuracy::test_support::{local_guarantee, satisfies}; -use asap_logical_asap_optimization::test_support::share_common_sub_dags; -use asap_types::ir::properties::AccuracyMetric; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchStatistic, SummaryInputExpr}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode}; -use support::{logical_candidates, lower_promql, physical_asap_dag}; - -/// The summary evaluation over a UnivMon state that `root` reads, if any. -fn univmon_estimate(root: &Rc) -> Option> { - OperatorNode::reachable(root).into_iter().find(|node| { - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &node.operator else { - return false; - }; - matches!(&summary_input.operator, Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::UnivMon) - }) -} - -/// The logical candidate of `query` that evaluates a UnivMon state. -fn candidate(query: &str, accuracy: AccuracyTarget) -> Rc { - let root = lower_promql(query, accuracy.clone()).unwrap(); - logical_candidates(root) - .into_iter() - .find_map(|node| univmon_estimate(&node)) - .expect("UnivMon candidate") -} - -#[test] -fn four_evaluations_size_their_value_frequency_states_and_keep_honest_guarantees() { - // Pass 1 sizes UnivMon per readout: the distinct count and L2 get their - // own shapes, while the total and entropy keep the baseline shape and so - // share one state. Sharing across readouts is the summary-capability - // rule's job. - let accuracy = AccuracyTarget::max_error(0.02); - let roots: Vec<_> = [ - ("distinct_over_time(m[5m])", accuracy.clone()), - ("count_over_time(m[5m])", accuracy.clone()), - ("l2_over_time(m[5m])", accuracy.clone()), - ("entropy_over_time(m[5m])", accuracy), - ] - .into_iter() - .enumerate() - .map(|(id, (query, accuracy))| (id, candidate(query, accuracy))) - .collect(); - let roots = share_common_sub_dags(roots); - let state = |index: usize| { - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = - &roots[index].1.operator - else { - panic!() - }; - Rc::clone(summary_input) - }; - assert!(Rc::ptr_eq(&state(1), &state(3)), "total and entropy share"); - for (a, b) in [(0, 1), (0, 2), (1, 2)] { - assert!( - !Rc::ptr_eq(&state(a), &state(b)), - "{a} and {b} are sized apart" - ); - } - for (index, root) in &roots { - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) = &root.operator - else { - panic!() - }; - let Operator::ASAP(ASAPOp::SummaryAgg { input, family, .. }) = &summary_input.operator - else { - panic!() - }; - assert!(matches!(input.item, Some(SummaryInputExpr::Column(_)))); - assert_eq!(input.weight, SummaryInputExpr::Constant(1.0)); - let guarantee = root.guarantee.as_ref(); - match index { - // The distinct count's failure probability waits for the - // source's distinct bound, which Stage 1 does not read. - 0 => { - let guarantee = guarantee.expect("a distinct-count guarantee"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!( - guarantee.spec.error.evaluate().map(|e| e <= 0.02), - Some(true) - ); - assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - assert!(local_guarantee(family, query).is_none()); - } - 1 => { - assert!(matches!( - query, - SketchStatistic::PointCount { value: None, .. } - )); - assert!(guarantee.is_some_and(|g| g.is_exact())); - } - 2 => { - let guarantee = guarantee.expect("an L2 guarantee"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(satisfies( - guarantee, - &AccuracyTarget::max_error(0.02).accuracy_metric(AccuracyMetric::Relative) - )); - assert!(guarantee.spec.failure_probability.evaluate().unwrap() <= 0.01); - } - // Entropy has no certified bound, so composition certifies nothing. - _ => { - assert!(guarantee.is_none()); - assert!(local_guarantee(family, query).is_none()); - } - } - physical_asap_dag(root); - } -} - -#[test] -fn uncalibrated_frequency_evaluations_do_not_bypass_accuracy_targets() { - // An unmeasured heuristic remains inspectable but is never certified or - // automatically selected for a caller-visible bounded-error result. - // L2 is certified; entropy is not. - for query in ["entropy_over_time(m[5m])"] { - for target in [ - AccuracyTarget::exact(), - AccuracyTarget::max_error(0.02), - AccuracyTarget::max_error(0.02).failure_probability(0.01), - ] { - let root = lower_promql(query, target.clone()).unwrap(); - let generated = logical_candidates(Rc::clone(&root)); - let run = run_all_stages(root, target.clone()); - let unknown: Vec<_> = generated - .iter() - .filter(|node| univmon_estimate(node).is_some() && node.guarantee.is_none()) - .collect(); - if target.is_exact() { - assert!(unknown.is_empty(), "{query}"); - } else { - assert!(!unknown.is_empty(), "{query}"); - let [asap_types::ir::QueryRoot::Operator(selected)] = run - .selected_plan - .as_deref() - .expect("a certified plan is selected") - else { - panic!("one operator result"); - }; - assert!(selected.guarantee.is_some(), "{query}"); - } - } - } -} diff --git a/crates/frontend-sql/src/frequency.rs b/crates/frontend-sql/src/frequency.rs index c95f4af1a..755e64514 100644 --- a/crates/frontend-sql/src/frequency.rs +++ b/crates/frontend-sql/src/frequency.rs @@ -4,8 +4,8 @@ //! unit counts. [`recognize_frequency_idioms`] names them as //! `FrequencyL2` / `FrequencyEntropy` intents, so Pass 1 can offer exact and //! summary candidates (UnivMon among them) for each. It preserves the -//! output schema, the predicates below the grouping, the query's accuracy -//! target, SQL's NULL result on empty input (decided by an exact population +//! output schema, the predicates below the grouping, SQL's NULL result on +//! empty input (decided by an exact population //! count, never by an estimate) and the entropy unit: the IR's entropy is in //! bits, the SQL idiom's `LN` gives nats. Integer products and nullable //! keys are refused, because SQL integer overflow and NULL groups are @@ -14,7 +14,6 @@ use asap_types::ir::operator::non_asap::any_measure_filtered; use asap_types::ir::operator::{ AggIntent, GroupKeys, JoinKind, Reduction, WindowFrameBound, WindowFrameOffset, WindowFuncKind, }; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::schema::DataType; use asap_types::ir::{ @@ -146,25 +145,18 @@ fn frequency_l2_rewrite(root: &Rc) -> Option> { ) { return None; } - let (key, accuracy, input) = grouped_unit_count(&inner)?; + let (key, input) = grouped_unit_count(&inner)?; sql_frequency_result( root, input, - AggIntent::FrequencyL2 { - col: Some(key), - // The count's target; Stage 1 sizes the norm's summary for it as - // a relative error. - accuracy, - }, + AggIntent::FrequencyL2 { col: Some(key) }, "frequency_l2", ScalarExpr::Column(0), ) } // Both frequency rules require a complete, unfiltered count per non-NULL identity. -fn grouped_unit_count( - node: &Rc, -) -> Option<(usize, AccuracyTarget, Rc)> { +fn grouped_unit_count(node: &Rc) -> Option<(usize, Rc)> { let NonASAPOp::Aggregate { reduction: Reduction::Reduce(keys), measures, @@ -179,7 +171,7 @@ fn grouped_unit_count( let [key] = keys.keys() else { return None; }; - let [AggIntent::Count { accuracy }] = measures.as_slice() else { + let [AggIntent::Count] = measures.as_slice() else { return None; }; let field = child.schema.fields.get(*key)?; @@ -193,7 +185,7 @@ fn grouped_unit_count( { return None; } - Some((*key, accuracy.clone(), Rc::clone(child))) + Some((*key, Rc::clone(child))) } fn probability_term(product: &ScalarExpr) -> Option<&ScalarExpr> { @@ -307,7 +299,7 @@ fn frequency_entropy_rewrite(root: &Rc) -> Option ) { return None; } - let (key, accuracy, input) = grouped_unit_count(child)?; + let (key, input) = grouped_unit_count(child)?; let nats = ScalarExpr::Arithmetic { op: ArithmeticOpKind::Mul, left: Box::new(ScalarExpr::Column(0)), @@ -329,12 +321,7 @@ fn frequency_entropy_rewrite(root: &Rc) -> Option sql_frequency_result( root, input, - AggIntent::FrequencyEntropy { - col: Some(key), - // The count's target; Stage 1 sizes the entropy's summary for it - // as an absolute error in bits. - accuracy, - }, + AggIntent::FrequencyEntropy { col: Some(key) }, "frequency_entropy", nats, ) @@ -364,12 +351,7 @@ fn sql_frequency_result( .ok() }; let statistic = aggregate(measure, name)?; - let count = aggregate( - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, - "population_count", - )?; + let count = aggregate(AggIntent::Count, "population_count")?; let child = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Join { kind: JoinKind::Cross, pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), diff --git a/crates/frontend-sql/src/lib.rs b/crates/frontend-sql/src/lib.rs index 9a8d09766..daa923eb3 100644 --- a/crates/frontend-sql/src/lib.rs +++ b/crates/frontend-sql/src/lib.rs @@ -17,7 +17,6 @@ pub mod test_support; use std::rc::Rc; use asap_frontend_common::resolve_root; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::OperatorNode; use asap_types::workload::SqlDialect; @@ -30,13 +29,9 @@ use sql::SqlLowerer; /// /// The `catalog` supplies table schemas (used both to plan the SQL with /// DataFusion and to carry positional column identity into the resolved -/// DAG). `accuracy` is threaded onto every approximate intent as it's built. -pub async fn lower_sql( - query: &str, - catalog: &SqlCatalog, - accuracy: AccuracyTarget, -) -> Result, SqlError> { - lower_sql_dialect(query, catalog, SqlDialect::DataFusionSQL, accuracy).await +/// DAG). +pub async fn lower_sql(query: &str, catalog: &SqlCatalog) -> Result, SqlError> { + lower_sql_dialect(query, catalog, SqlDialect::DataFusionSQL).await } /// Lower a single SQL query string under an explicit [`SqlDialect`]. @@ -51,10 +46,9 @@ pub async fn lower_sql_dialect( query: &str, catalog: &SqlCatalog, dialect: SqlDialect, - accuracy: AccuracyTarget, ) -> Result, SqlError> { let unresolved = SqlLowerer::with_dialect(catalog, dialect) - .lower(query, &accuracy) + .lower(query) .await?; // Binding resolves names and derives every node's schema; result-type // checks (such as temporal subtraction, whose duration unit the IR cannot diff --git a/crates/frontend-sql/src/sql/collection_udfs.rs b/crates/frontend-sql/src/sql/collection_udfs.rs index 17711eb83..c4892190c 100644 --- a/crates/frontend-sql/src/sql/collection_udfs.rs +++ b/crates/frontend-sql/src/sql/collection_udfs.rs @@ -118,7 +118,7 @@ impl CollectionUdf { Ok(ScalarExpr::Column(index)) } }) - .collect::>>()?; + .collect::>>()?; match self.function { CollectionFunction::Element => element_access_type(&args, &schema), CollectionFunction::StructField => struct_field_type(&args, &schema), diff --git a/crates/frontend-sql/src/sql/mod.rs b/crates/frontend-sql/src/sql/mod.rs index 4121dd22e..b015c0032 100644 --- a/crates/frontend-sql/src/sql/mod.rs +++ b/crates/frontend-sql/src/sql/mod.rs @@ -59,7 +59,7 @@ use asap_frontend_common::{ }; use asap_sql_function_catalog::{AggSemantic, Arity, RewriteKind}; use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::operator::operator_properties::{ +use asap_types::ir::operator::arguments::{ GroupKeys, Reduction, Source, WindowFrame, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, }; @@ -67,7 +67,6 @@ use asap_types::ir::schema::{DataType, FieldDataType, Schema}; use asap_types::ir::TimeRangeKind; use asap_types::ir::operator::{JoinKind, RelationalSetOpKind, WindowFuncKind}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::workload::SqlDialect; @@ -89,9 +88,6 @@ use self::types::{arrow_to_dtype, scalar_value_to_asap, schema_to_arrow}; pub(crate) struct SqlLowerer<'a> { catalog: &'a SqlCatalog, dialect: SqlDialect, - /// The target [`lower`](Self::lower) threads onto every approximate - /// intent; exact until a `lower` call sets it. - accuracy: AccuracyTarget, } impl<'a> SqlLowerer<'a> { @@ -109,16 +105,10 @@ impl<'a> SqlLowerer<'a> { /// still unknown to DataFusion's planner and still fails there, and /// `ElasticSQL` has no vendored parser at all. pub(crate) fn with_dialect(catalog: &'a SqlCatalog, dialect: SqlDialect) -> Self { - Self { - catalog, - dialect, - accuracy: AccuracyTarget::exact(), - } + Self { catalog, dialect } } - /// Parse + lower a SQL query to the name-based tree, threading - /// `accuracy` onto every approximate intent (`Count`, `Quantile`, - /// `Cardinality`) as it is built. + /// Parse + lower a SQL query to the name-based tree. /// /// Runs `ApplyFunctionRewrites` — the single `AnalyzerRule` DataFusion's /// own `Analyzer` uses internally to apply `FunctionRewrite`s, called @@ -140,11 +130,7 @@ impl<'a> SqlLowerer<'a> { /// check would. Going straight to `ApplyFunctionRewrites` avoids that /// entirely. TypeCoercion then records implicit conversions explicitly, /// including timestamp literals in predicates, before IR validation. - pub(crate) async fn lower( - &mut self, - sql: &str, - accuracy: &AccuracyTarget, - ) -> Result { + pub(crate) async fn lower(&self, sql: &str) -> Result { let ctx = self.build_context()?; let state = ctx.state(); let statement = if matches!(self.dialect, SqlDialect::ClickhouseSQL) { @@ -201,7 +187,6 @@ impl<'a> SqlLowerer<'a> { let plan = datafusion::optimizer::analyzer::type_coercion::TypeCoercion::new() .analyze(plan, &ctx.state().options())?; let plan = drop_integer_decimal_widening(plan)?; - self.accuracy = accuracy.clone(); self.lower_plan(&plan) } @@ -866,7 +851,7 @@ impl<'a> SqlLowerer<'a> { .collect(); let measures = aggr_expr .iter() - .map(|e| lower_agg_intent(e, &self.accuracy)) + .map(lower_agg_intent) .collect::, LoweringError>>()?; // Empty when nothing is filtered — the one canonical unfiltered shape. let filters = if measure_filters.iter().any(Option::is_some) { @@ -1113,7 +1098,7 @@ impl<'a> SqlLowerer<'a> { .collect::, LoweringError>>()?; let measures = aggr_expr .iter() - .map(|e| lower_agg_intent(e, &self.accuracy)) + .map(lower_agg_intent) .collect::, LoweringError>>()?; let input = derived.wrap(input)?; @@ -1708,17 +1693,14 @@ fn conditional_count_arm(expr: &Expr) -> Option<(&Expr, &Expr)> { /// (issue #225) as flat data (`NATIVE_FUNCTIONS`); what stays here is /// call-site logic that isn't a function of the name alone — the DISTINCT /// modifier rule, the "reducer argument must be a bare column" rule -/// (`reducer_col`), φ extraction from a literal argument, and attaching -/// `accuracy` to approximate intents. `resolve_root` resolves `col` to a positional +/// (`reducer_col`) and φ extraction from a literal argument. `resolve_root` +/// resolves `col` to a positional /// `ColumnId`; the output name (DataFusion's own, e.g. /// `"sum(metrics.bytes)"`) is threaded separately as `Aggregate.output_names`, /// not carried here. -fn lower_agg_intent( - expr: &Expr, - accuracy: &AccuracyTarget, -) -> Result, LoweringError> { +fn lower_agg_intent(expr: &Expr) -> Result, LoweringError> { match expr { - Expr::Alias(a) => lower_agg_intent(&a.expr, accuracy), + Expr::Alias(a) => lower_agg_intent(&a.expr), Expr::AggregateFunction(agg_fn) => { let name = agg_fn.func.name().to_lowercase(); // ClickHouse's row-selecting `argMax`/`argMin` — `RewriteKind:: @@ -1782,12 +1764,9 @@ fn lower_agg_intent( } args => AggIntent::Cardinality { cols: args.iter().map(distinct_col).collect::>()?, - accuracy: accuracy.clone(), }, }, - AggSemantic::Count => AggIntent::Count { - accuracy: accuracy.clone(), - }, + AggSemantic::Count => AggIntent::Count, AggSemantic::Sum => AggIntent::Sum { col: col(&agg_fn.params.args)?, }, @@ -1827,11 +1806,9 @@ fn lower_agg_intent( _ => extract_percentile_q(&agg_fn.params.args)?, }, }, - accuracy: accuracy.clone(), }, AggSemantic::Cardinality => AggIntent::Cardinality { cols: vec![reducer_col(&name, &agg_fn.params.args)?], - accuracy: accuracy.clone(), }, }) } diff --git a/crates/frontend-sql/src/test_support.rs b/crates/frontend-sql/src/test_support.rs index c8de29b9c..aa599d562 100644 --- a/crates/frontend-sql/src/test_support.rs +++ b/crates/frontend-sql/src/test_support.rs @@ -1,6 +1,5 @@ //! Lowering steps only tests call, exposed under the `test-support` feature. use asap_frontend_common::UnresolvedOp; -use asap_types::ir::properties::AccuracyTarget; use crate::sql::{SqlCatalog, SqlLowerer}; use crate::SqlError; @@ -8,10 +7,6 @@ use crate::SqlError; /// `query` lowered to the name-based tree, parsed as /// `SqlDialect::DataFusionSQL`, before `resolve_root` binds it and before /// frequency recognition. -pub async fn lower_unresolved( - query: &str, - catalog: &SqlCatalog, - accuracy: &AccuracyTarget, -) -> Result { - SqlLowerer::new(catalog).lower(query, accuracy).await +pub async fn lower_unresolved(query: &str, catalog: &SqlCatalog) -> Result { + SqlLowerer::new(catalog).lower(query).await } diff --git a/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs b/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs index 7185e8f8a..73312a7c2 100644 --- a/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs +++ b/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs @@ -39,7 +39,6 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError as LoweringError}; use asap_types::ir::operator::{AggIntent, GroupKeys}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::workload::SqlDialect; @@ -96,13 +95,7 @@ fn queries() -> Vec { } async fn lower(q: &str) -> Result, LoweringError> { - lower_sql_dialect( - q, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await + lower_sql_dialect(q, &catalog(), SqlDialect::ClickhouseSQL).await } /// Pinned outcome for one corpus query. @@ -258,7 +251,7 @@ async fn top_k_queries_are_count_grouped_by_prefix() { idx + 1 ); assert!( - matches!(measures.as_slice(), [AggIntent::Count { .. }]), + matches!(measures.as_slice(), [AggIntent::Count]), "q{} ({label}) expected a single Count aggregate, got {measures:?}", idx + 1 ); diff --git a/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs b/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs index bc7353f9e..9975ebb19 100644 --- a/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs +++ b/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs @@ -14,7 +14,6 @@ //! tally** by outcome category -- see the module doc on [`Category`] for why. use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::workload::SqlDialect; use datafusion::error::DataFusionError; @@ -71,13 +70,7 @@ fn catalog() -> SqlCatalog { } async fn lower(q: &str) -> Result, SqlError> { - lower_sql_dialect( - q, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await + lower_sql_dialect(q, &catalog(), SqlDialect::ClickhouseSQL).await } /// Coarse outcome bucket for a corpus query. Deliberately coarser than diff --git a/crates/frontend-sql/tests/clickbench/clickbench.rs b/crates/frontend-sql/tests/clickbench/clickbench.rs index c57b15017..5dfbbec25 100644 --- a/crates/frontend-sql/tests/clickbench/clickbench.rs +++ b/crates/frontend-sql/tests/clickbench/clickbench.rs @@ -19,7 +19,6 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError as LoweringError}; use asap_types::ir::operator::{AggIntent, GroupKeys}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::workload::SqlDialect; @@ -166,13 +165,7 @@ fn queries() -> Vec<&'static str> { } async fn lower(q: &str) -> Result, LoweringError> { - lower_sql_dialect( - q, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await + lower_sql_dialect(q, &catalog(), SqlDialect::ClickhouseSQL).await } /// Every ClickBench query lowers end to end. A query that stops lowering @@ -251,7 +244,7 @@ async fn top_search_phrases_canonicalize_to_top_k() { let (keys, inner) = first_aggregate(child).expect("q13 expected a per-phrase Aggregate"); assert_eq!(*keys, GroupKeys::by(vec![column("SearchPhrase")])); assert!( - matches!(inner.as_slice(), [AggIntent::Count { .. }]), + matches!(inner.as_slice(), [AggIntent::Count]), "q13 expected a per-phrase Count, got {inner:?}" ); } diff --git a/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs b/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs index 53ffbbfed..aae70b05a 100644 --- a/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs +++ b/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs @@ -23,7 +23,6 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog, SqlError as LoweringError}; use asap_types::ir::operator::{AggIntent, GroupKeys}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; @@ -125,7 +124,7 @@ fn has_window_func(node: &OperatorNode) -> bool { } async fn lower(q: &str) -> Rc { - lower_sql(q, &catalog(), AccuracyTarget::exact()) + lower_sql(q, &catalog()) .await .unwrap_or_else(|e| panic!("expected {q:?} to lower, got error: {e}")) } @@ -145,7 +144,7 @@ async fn corpus_lowering_covers_every_query() { let mut t = Tally::default(); for q in queries() { // A panic here (not an `Err`) fails the test — the totality guarantee. - match lower_sql(&q, &cat, AccuracyTarget::exact()).await { + match lower_sql(&q, &cat).await { Ok(_) => t.lowered += 1, // DataFusion surfaces parse/plan failures as `DataFusion(_)`. Err(LoweringError::DataFusion(_)) => t.unparseable += 1, @@ -183,7 +182,7 @@ async fn packet_count_is_count_intent() { let qe = lower("SELECT COUNT(*) AS total_packets FROM packets").await; let (by, measures) = first_aggregate(&qe).expect("expected an Aggregate"); assert!(by.is_empty(), "global count has no grouping"); - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } #[tokio::test] @@ -224,7 +223,7 @@ async fn five_tuple_flow_group_by_binds_five_keys() { .await; let (by, measures) = first_aggregate(&qe).expect("expected an Aggregate"); assert_eq!(by.len(), 5, "grouped by the 5-tuple"); - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } #[tokio::test] diff --git a/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs b/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs index a7ad616f1..f7965edf5 100644 --- a/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs +++ b/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs @@ -23,7 +23,6 @@ //! which is the same narrowing sidra's own catalog file makes. use asap_frontend_sql::{lower_sql, SqlCatalog, SqlError as LoweringError}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; const CORPUS: &str = include_str!("data/tpch_deequ_queries.sql"); @@ -85,7 +84,7 @@ async fn lowers_the_warehouse_ingestion_check_set() { let mut rejected = Vec::new(); let mut lowered = 0; for (id, query) in queries { - match lower_sql(query, &cat, AccuracyTarget::exact()).await { + match lower_sql(query, &cat).await { Ok(_) => lowered += 1, Err(LoweringError::UnsupportedAggregate(reason)) => rejected.push((id, reason)), Err(error) => panic!("unexpected failure for U-{id}: {error}"), diff --git a/crates/frontend-sql/tests/frequency_entropy.rs b/crates/frontend-sql/tests/frequency_entropy.rs index 7bd9962db..3c8d66973 100644 --- a/crates/frontend-sql/tests/frequency_entropy.rs +++ b/crates/frontend-sql/tests/frequency_entropy.rs @@ -4,7 +4,6 @@ use asap_frontend_common::resolve_root; use asap_frontend_sql::test_support::lower_unresolved; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; @@ -32,15 +31,8 @@ fn has_entropy(node: &OperatorNode) -> bool { #[tokio::test] async fn recognizes_sql_entropy_in_nats() { let sql = "SELECT -SUM(p * LN(p)) AS entropy FROM (SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f"; - let rewritten = lower_sql(sql, &catalog(false), AccuracyTarget::exact()) - .await - .unwrap(); - let original = resolve_root( - &lower_unresolved(sql, &catalog(false), &AccuracyTarget::exact()) - .await - .unwrap(), - ) - .unwrap(); + let rewritten = lower_sql(sql, &catalog(false)).await.unwrap(); + let original = resolve_root(&lower_unresolved(sql, &catalog(false)).await.unwrap()).unwrap(); assert!(has_entropy(&rewritten)); assert_eq!(original.schema, rewritten.schema); assert!(!has_entropy(&original)); @@ -59,7 +51,7 @@ async fn declines_non_equivalent_entropy_shapes() { ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip HAVING COUNT(*) > 1) f", false), ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", true), ] { - let root = lower_sql(sql, &catalog(nullable), AccuracyTarget::exact()) + let root = lower_sql(sql, &catalog(nullable)) .await .unwrap(); assert!(!has_entropy(&root), "{sql}"); @@ -70,8 +62,7 @@ async fn declines_non_equivalent_entropy_shapes() { /// is an exact count. #[tokio::test] async fn entropy_accuracy_and_population_guard_are_separate() { - let target = AccuracyTarget::max_error(0.05).failure_probability(0.01); - let node = lower_sql("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", &catalog(false), target.clone()).await.unwrap(); + let node = lower_sql("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", &catalog(false)).await.unwrap(); let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { panic!("project"); }; @@ -81,14 +72,9 @@ async fn entropy_accuracy_and_population_guard_are_separate() { let NonASAPOp::Aggregate { measures, .. } = left.expect_non_asap() else { panic!("entropy"); }; - assert!( - matches!(&measures[0], AggIntent::FrequencyEntropy { accuracy, .. } if *accuracy == target) - ); + assert!(matches!(&measures[0], AggIntent::FrequencyEntropy { .. })); let NonASAPOp::Aggregate { measures, .. } = right.expect_non_asap() else { panic!("count"); }; - assert!(matches!( - measures.as_slice(), - [AggIntent::Count { accuracy }] if accuracy.is_exact() - )); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } diff --git a/crates/frontend-sql/tests/frequency_l2.rs b/crates/frontend-sql/tests/frequency_l2.rs index dbf709609..bd8261de3 100644 --- a/crates/frontend-sql/tests/frequency_l2.rs +++ b/crates/frontend-sql/tests/frequency_l2.rs @@ -4,7 +4,6 @@ use asap_frontend_common::resolve_root; use asap_frontend_sql::test_support::lower_unresolved; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::stages::HowToComputeAggregateMeasure; @@ -34,12 +33,7 @@ fn has_l2(node: &OperatorNode) -> bool { /// The relational DAG the frontend lowers before frequency recognition. async fn relational(sql: &str, catalog: &SqlCatalog) -> Rc { - resolve_root( - &lower_unresolved(sql, catalog, &AccuracyTarget::exact()) - .await - .unwrap(), - ) - .unwrap() + resolve_root(&lower_unresolved(sql, catalog).await.unwrap()).unwrap() } const CAST_PRODUCT: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) AS norm FROM (SELECT src_ip, COUNT(*) AS c FROM flows WHERE keep GROUP BY src_ip) f"; @@ -50,9 +44,7 @@ const CAST_COUNT: &str = "SELECT SQRT(SUM(c * c)) AS norm FROM (SELECT src_ip, C #[tokio::test] async fn recognizes_float_frequency_l2() { for sql in [CAST_PRODUCT, CAST_COUNT] { - let rewritten = lower_sql(sql, &catalog(false), AccuracyTarget::exact()) - .await - .unwrap(); + let rewritten = lower_sql(sql, &catalog(false)).await.unwrap(); let original = relational(sql, &catalog(false)).await; assert!(has_l2(&rewritten), "{sql}"); assert_eq!(original.schema, rewritten.schema); @@ -71,7 +63,7 @@ async fn declines_non_equivalent_frequency_shapes() { ("SELECT SQRT(SUM(c*c)) FROM (SELECT src_ip, CAST(COUNT(*) AS DOUBLE) AS c FROM flows GROUP BY src_ip HAVING COUNT(*) > 1) f", false), ("SELECT SQRT(SUM(c*c)) FROM (SELECT src_ip, CAST(SUM(CAST(keep AS BIGINT)) AS DOUBLE) AS c FROM flows GROUP BY src_ip) f", false), ] { - let root = lower_sql(sql, &catalog(nullable), AccuracyTarget::exact()) + let root = lower_sql(sql, &catalog(nullable)) .await .unwrap(); assert!(!has_l2(&root), "{sql}"); @@ -81,10 +73,7 @@ async fn declines_non_equivalent_frequency_shapes() { /// The L2 intent carries the query's accuracy target, not a default. #[tokio::test] async fn frequency_l2_preserves_accuracy_target() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let node = lower_sql(CAST_COUNT, &catalog(false), target.clone()) - .await - .unwrap(); + let node = lower_sql(CAST_COUNT, &catalog(false)).await.unwrap(); let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { panic!("project"); }; @@ -94,17 +83,14 @@ async fn frequency_l2_preserves_accuracy_target() { let NonASAPOp::Aggregate { measures, .. } = left.expect_non_asap() else { panic!("aggregate"); }; - assert!(matches!(&measures[0], AggIntent::FrequencyL2 { accuracy, .. } if *accuracy == target)); + assert!(matches!(&measures[0], AggIntent::FrequencyL2 { .. })); } /// An approximate L2 estimate never decides whether SQL returns NULL: the /// empty-input guard is an exact count. #[tokio::test] async fn frequency_empty_input_guard_uses_an_exact_count() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let node = lower_sql(CAST_COUNT, &catalog(false), target) - .await - .unwrap(); + let node = lower_sql(CAST_COUNT, &catalog(false)).await.unwrap(); let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { panic!("project"); }; @@ -114,10 +100,7 @@ async fn frequency_empty_input_guard_uses_an_exact_count() { let NonASAPOp::Aggregate { measures, .. } = right.expect_non_asap() else { panic!("count"); }; - assert!(matches!( - measures.as_slice(), - [AggIntent::Count { accuracy }] if accuracy.is_exact() - )); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } /// Stage 1 offers the exact option and summary candidates for the @@ -126,14 +109,14 @@ async fn frequency_empty_input_guard_uses_an_exact_count() { async fn stage1_offers_exact_and_summary_l2_candidates() { use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; use asap_types::ir::QueryRoot; - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let root = lower_sql(CAST_COUNT, &catalog(false), target) - .await - .unwrap(); + let root = lower_sql(CAST_COUNT, &catalog(false)).await.unwrap(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); let l2 = inventory @@ -179,11 +162,7 @@ async fn example2_queries_lower_to_frequency_intents() { let q3_int = format!("SELECT SQRT(SUM(c * c)) FROM (SELECT src_ip, COUNT(*) AS c FROM flows {WINDOW} GROUP BY src_ip)"); let lower = |sql: String| { let catalog = &catalog; - async move { - lower_sql(&sql, catalog, AccuracyTarget::exact()) - .await - .unwrap() - } + async move { lower_sql(&sql, catalog).await.unwrap() } }; let q2 = lower(q2).await; assert!( diff --git a/crates/frontend-sql/tests/netflow/netflow.rs b/crates/frontend-sql/tests/netflow/netflow.rs index 596f911dd..033dcf153 100644 --- a/crates/frontend-sql/tests/netflow/netflow.rs +++ b/crates/frontend-sql/tests/netflow/netflow.rs @@ -8,7 +8,6 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::{AggIntent, GroupKeys}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; @@ -113,7 +112,7 @@ async fn netflow_sql_corpus_lowers_to_expected_intents() { for (idx, (query, expected)) in queries.iter().zip(EXPECTED).enumerate() { // A successful `lower_sql` already derived every node's schema. - let qe = lower_sql(query, &catalog(), AccuracyTarget::exact()) + let qe = lower_sql(query, &catalog()) .await .unwrap_or_else(|err| panic!("q{} failed to lower:\n{query}\n{err}", idx + 1)); assert!( @@ -146,7 +145,7 @@ fn assert_expected(qe: &Rc, expected: Expected, case_no: usize) { "q{case_no} expected count-ranked TopK({k}), got {qe:?}" ); assert!( - aggregate_by_with(qe, inner_by, |agg| matches!(agg, AggIntent::Count { .. })), + aggregate_by_with(qe, inner_by, |agg| matches!(agg, AggIntent::Count)), "q{case_no} expected inner Count grouped by {inner_by:?}, got {qe:?}" ); } diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index c6f31079a..dce78d103 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -3,12 +3,8 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{ - apply_materialization_timings, physical_export::compile_physical_asap_dag, - MaterializationAssignment, NodeToTimedNodeCache, NonASAPOp, OperatorNode, ScalarExpr, -}; +use asap_types::ir::{NonASAPOp, OperatorNode, ScalarExpr}; fn catalog() -> SqlCatalog { let schema = Schema::new(vec![ @@ -22,9 +18,7 @@ fn catalog() -> SqlCatalog { } async fn lower(sql: &str) -> Rc { - lower_sql(sql, &catalog(), AccuracyTarget::exact()) - .await - .unwrap() + lower_sql(sql, &catalog()).await.unwrap() } fn aggregate(query: &OperatorNode) -> (&[AggIntent], &OperatorNode) { @@ -114,12 +108,7 @@ async fn corr_rejects_unrepresented_forms() { "SELECT corr(x, y) OVER () FROM a", "SELECT corr(x) FROM a", ] { - assert!( - lower_sql(sql, &catalog(), AccuracyTarget::exact()) - .await - .is_err(), - "{sql}" - ); + assert!(lower_sql(sql, &catalog()).await.is_err(), "{sql}"); } } @@ -143,47 +132,3 @@ async fn corr_filter_is_a_measure_filter() { filters(&query) ); } - -// The plan the planner pipeline selects retains the complete typed query and -// compiles to a physical ASAP DAG. -#[tokio::test] -async fn corr_survives_exact_plan_compilation() { - let query = lower("SELECT corr(x, y) AS r FROM a").await; - // `a` holds 60,000 rows: the query scans all of them. - let data: asap_types::workload::DataWorkload = serde_json::from_value( - serde_json::json!({"arrival": "at_rest", "sources": {"a": {"rows_held": 60000}}}), - ) - .unwrap(); - let run = asap_planner::run_planning( - vec![asap_types::ir::QueryRoot::Operator(query.clone())], - &[asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: asap_types::workload::QueryRequirements { - accuracy: asap_types::ir::properties::AccuracyTarget::exact(), - max_latency_ms: None, - }, - predictability: Default::default(), - recurrence: Default::default(), - range_evaluation: None, - }], - &data, - asap_planner::PlanningModels::builtin(), - asap_planner::PlanSearchSettings::default(), - ) - .unwrap(); - let [asap_types::ir::QueryRoot::Operator(plan)] = run.selected_plan.as_deref().unwrap() else { - panic!("one operator root"); - }; - assert!(plan.guarantee.as_ref().unwrap().is_exact()); - // Pass 1 has no candidate for `corr`: the plan is the query's own - // operator DAG, no ASAP node added. - assert!(!plan.contains_asap(), "expected the exact plan"); - assert_eq!(aggregate(plan).0, aggregate(&query).0); - let timed = apply_materialization_timings( - plan, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap(); -} diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index ad9ba1367..84829c04b 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -16,7 +16,6 @@ use asap_types::ir::operator::{ AggIntent, GroupKeys, JoinKind, Reduction, Source, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, WindowFuncKind, }; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ExprSemantics, NonASAPOp, OperatorNode, ScalarExpr}; @@ -52,7 +51,7 @@ fn catalog() -> SqlCatalog { } async fn lower(sql: &str) -> Rc { - lower_sql(sql, &catalog(), AccuracyTarget::exact()) + lower_sql(sql, &catalog()) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) } @@ -64,7 +63,7 @@ fn op(node: &OperatorNode) -> &NonASAPOp { #[tokio::test] async fn planning_subquery_bridge_rejects_a_relation_without_vector_conversion() { - let result = lower_sql("SELECT max(value) FROM (SELECT asap_promql_subquery(21600000, 60000) AS value FROM (SELECT sum(bytes) AS value FROM metrics))", &catalog(), AccuracyTarget::exact()).await; + let result = lower_sql("SELECT max(value) FROM (SELECT asap_promql_subquery(21600000, 60000) AS value FROM (SELECT sum(bytes) AS value FROM metrics))", &catalog()).await; assert!(result.is_err()); } @@ -98,7 +97,6 @@ async fn planning_relation_bridges_reject_ambiguous_shapes() { let missing_alias = lower_sql( "SELECT asap_promql_subquery(300000, 60000) FROM metrics", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap_err(); @@ -107,7 +105,6 @@ async fn planning_relation_bridges_reject_ambiguous_shapes() { let histogram_with_extra_column = lower_sql( "SELECT service, asap_histogram_quantile(0.95) AS value FROM metrics", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap_err(); @@ -118,7 +115,6 @@ async fn planning_relation_bridges_reject_ambiguous_shapes() { let invalid_q = lower_sql( "SELECT asap_histogram_quantile(1.5) AS value FROM metrics", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap_err(); @@ -314,7 +310,7 @@ async fn count_ranked_topk_is_heavy_hitter() { let (inner_by, inner_measures) = find_aggregate(child).expect("expected inner Count aggregate"); assert_eq!(inner_by, &vec![1], "inner Count grouped by service → col 1"); assert!( - matches!(inner_measures.as_slice(), [AggIntent::Count { .. }]), + matches!(inner_measures.as_slice(), [AggIntent::Count]), "inner aggregate is the explicit Count, got {inner_measures:?}" ); } @@ -366,12 +362,12 @@ async fn filtered_count_ranking_is_not_a_heavy_hitter() { GROUP BY service ORDER BY c DESC LIMIT 10", "SELECT service, COUNT(bytes) AS c FROM metrics GROUP BY service ORDER BY c DESC LIMIT 10", ] { - let qe = lower_sql(sql, &catalog, AccuracyTarget::exact()) + let qe = lower_sql(sql, &catalog) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); assert!( - matches!(measures.as_slice(), [AggIntent::Count { .. }]), + matches!(measures.as_slice(), [AggIntent::Count]), "{sql}: the filtered Count must stay, got {measures:?}" ); let NonASAPOp::Aggregate { filters, .. } = op(find_aggregate_node(&qe).unwrap()) else { @@ -404,12 +400,7 @@ async fn non_count_ranked_limit_keeps_the_aggregate() { async fn distinct_value_reducer_is_rejected_not_dropped() { // The canonical intent algebra has no distinct-Sum; SUM(DISTINCT x) must // be rejected, not silently lowered as SUM(x). - let res = lower_sql( - "SELECT SUM(DISTINCT bytes) FROM metrics", - &catalog(), - AccuracyTarget::exact(), - ) - .await; + let res = lower_sql("SELECT SUM(DISTINCT bytes) FROM metrics", &catalog()).await; assert!(res.is_err(), "SUM(DISTINCT ...) should be rejected"); } @@ -438,7 +429,7 @@ async fn count_star_is_count_intent() { let qe = lower("SELECT COUNT(*) FROM metrics").await; let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); assert!(by.is_empty()); - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } #[tokio::test] @@ -737,7 +728,6 @@ async fn a_multi_column_in_subquery_is_rejected() { let err = lower_sql( "SELECT service FROM metrics WHERE service IN (SELECT service, region FROM hosts)", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("IN must select one column"); @@ -962,7 +952,6 @@ async fn groups_frame_is_rejected() { "SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts \ GROUPS BETWEEN 2 PRECEDING AND CURRENT ROW) FROM metrics", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("GROUPS frame unit must be rejected"); @@ -1167,7 +1156,6 @@ async fn a_correlated_in_subquery_is_rejected() { "SELECT service FROM metrics m WHERE service IN \ (SELECT h.service FROM hosts h WHERE h.region = m.service)", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("correlated IN needs both a key match and a correlation"); @@ -1179,7 +1167,7 @@ async fn a_correlated_in_subquery_is_rejected() { /// The name-based tree, before `resolve_root`. async fn lower_unresolved(sql: &str) -> UnresolvedOp { let catalog = catalog(); - asap_frontend_sql::test_support::lower_unresolved(sql, &catalog, &AccuracyTarget::exact()) + asap_frontend_sql::test_support::lower_unresolved(sql, &catalog) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) } @@ -1451,27 +1439,6 @@ async fn median_is_the_same_intent_as_an_explicit_half_percentile() { assert_eq!(m_measures, p_measures); } -#[tokio::test] -async fn median_threads_the_accuracy_target() { - // The `approx_` prefix does not decide: the AccuracyTarget does. - let qe = lower_sql( - "SELECT approx_median(latency) FROM metrics", - &catalog(), - AccuracyTarget::max_error(0.01), - ) - .await - .expect("approx_median should lower"); - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - matches!( - measures.as_slice(), - [AggIntent::Quantile { accuracy, .. }] - if (accuracy.max_error - 0.01).abs() < 1e-12 - ), - "median must carry the workload's accuracy target, got {measures:?}" - ); -} - #[tokio::test] async fn median_over_an_expression_binds_the_derived_column() { // Was rejected when filed (#111); supported since #110 materialized the @@ -1708,7 +1675,6 @@ async fn grouping_function_is_rejected() { let err = lower_sql( "SELECT service, SUM(bytes), GROUPING(service) FROM metrics GROUP BY ROLLUP(service)", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("GROUPING() must be rejected while __grouping_id is dropped"); @@ -1722,7 +1688,6 @@ async fn a_non_column_key_inside_a_grouping_set_is_rejected() { let err = lower_sql( "SELECT date_trunc('minute', ts) AS m, SUM(bytes) FROM metrics GROUP BY ROLLUP(m)", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("expression key inside ROLLUP must be rejected"); @@ -1766,7 +1731,6 @@ async fn an_ambiguous_passthrough_column_is_rejected_only_when_projecting() { "SELECT m.service, h.service, SUM(m.bytes) FROM metrics m \ JOIN hosts h ON m.service = h.service GROUP BY m.service, h.service", &catalog(), - AccuracyTarget::exact(), ) .await; assert!( @@ -1778,7 +1742,6 @@ async fn an_ambiguous_passthrough_column_is_rejected_only_when_projecting() { "SELECT m.service, h.service, SUM(m.bytes * 2) FROM metrics m \ JOIN hosts h ON m.service = h.service GROUP BY m.service, h.service", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("ambiguous passthrough must be rejected, not silently resolved"); @@ -1798,13 +1761,9 @@ async fn array_agg_is_deliberately_rejected() { // feed the sketch path. // // This test exists so the rejection reads as a decision rather than a gap. - let err = lower_sql( - "SELECT array_agg(service) FROM metrics", - &catalog(), - AccuracyTarget::exact(), - ) - .await - .expect_err("array_agg must not lower to an intent"); + let err = lower_sql("SELECT array_agg(service) FROM metrics", &catalog()) + .await + .expect_err("array_agg must not lower to an intent"); assert!( format!("{err}").contains("unsupported aggregate: array_agg"), "expected a clean UnsupportedAggregate, got {err}" @@ -1829,7 +1788,6 @@ async fn clickhouse_identifiers_keep_their_case() { "SELECT RegionID, COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID", &hits, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap_or_else(|e| panic!("mixed-case ClickHouse columns must resolve: {e}")); @@ -1840,14 +1798,9 @@ async fn clickhouse_identifiers_keep_their_case() { "expected COUNT(DISTINCT UserID) over column 1, got {measures:?}" ); - let err = lower_sql_dialect( - "SELECT userid FROM hits", - &hits, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect_err("ClickHouse must not match `userid` to `UserID`"); + let err = lower_sql_dialect("SELECT userid FROM hits", &hits, SqlDialect::ClickhouseSQL) + .await + .expect_err("ClickHouse must not match `userid` to `UserID`"); assert!( format!("{err}").contains("userid"), "unexpected error: {err}" @@ -1860,14 +1813,9 @@ async fn clickhouse_identifiers_keep_their_case() { // uniqExact from #221) ─────────────────────────────────────────────────── async fn lower_clickhouse(sql: &str) -> Rc { - lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) + lower_sql_dialect(sql, &catalog(), SqlDialect::ClickhouseSQL) + .await + .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) } fn temporal_aggregate(node: &OperatorNode) -> (&AggIntent, std::time::Duration, &OperatorNode) { @@ -1923,14 +1871,9 @@ async fn temporal_aggregate_rejects_non_timestamp_and_non_positive_window() { "SELECT asap_rate(latency, ts, 0) FROM metrics", "SELECT asap_rate(latency, ts, bytes) FROM metrics", ] { - let err = lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect_err("invalid temporal arguments must fail closed"); + let err = lower_sql_dialect(sql, &catalog(), SqlDialect::ClickhouseSQL) + .await + .expect_err("invalid temporal arguments must fail closed"); assert!( format!("{err}").contains("timestamp argument") || format!("{err}").contains("window_ms"), @@ -1945,7 +1888,6 @@ async fn temporal_aggregate_rejects_mixed_reducers() { "SELECT asap_rate(latency, ts, 300000), sum(bytes) FROM metrics", &catalog(), SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect_err("one child cannot carry temporal and ordinary aggregate semantics"); @@ -1958,7 +1900,6 @@ async fn last_fails_closed_until_an_executable_summary_exists() { "SELECT service, asap_last(latency, ts, 300000) FROM metrics GROUP BY service", &catalog(), SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect_err("last must not be advertised without an executable physical summary"); @@ -1984,14 +1925,9 @@ async fn temporal_grouping_requires_the_complete_declared_series_identity() { "SELECT asap_rate(value, ts, 300000) FROM samples", "SELECT service, asap_rate(value, ts, 300000) FROM samples GROUP BY service", ] { - let err = lower_sql_dialect( - sql, - &multi_series, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect_err("partial identity must not merge counter series"); + let err = lower_sql_dialect(sql, &multi_series, SqlDialect::ClickhouseSQL) + .await + .expect_err("partial identity must not merge counter series"); assert!(format!("{err}").contains("declared series identity")); } @@ -2000,7 +1936,6 @@ async fn temporal_grouping_requires_the_complete_declared_series_identity() { FROM samples GROUP BY service, instance", &multi_series, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect("the complete declared series identity is safe"); @@ -2021,7 +1956,6 @@ async fn temporal_grouping_requires_the_complete_declared_series_identity() { "SELECT service, asap_rate(value, ts, 300000) FROM samples GROUP BY service", &row_id_only, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect_err("a row key without time does not prove a series identity"); @@ -2035,14 +1969,9 @@ async fn temporal_grouping_rejects_value_time_and_duplicate_resolved_columns() { "SELECT m.service, asap_rate(m.latency, m.ts, 300000) \ FROM metrics m GROUP BY m.service, service", ] { - let err = lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect_err("unsafe or duplicate resolved grouping must fail closed"); + let err = lower_sql_dialect(sql, &catalog(), SqlDialect::ClickhouseSQL) + .await + .expect_err("unsafe or duplicate resolved grouping must fail closed"); let message = format!("{err}"); assert!( message.contains("timestamp or value") @@ -2231,7 +2160,6 @@ async fn arg_max_rejects_a_non_column_argument() { "SELECT argMax(service, latency * 2) FROM metrics", &catalog(), SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect_err("argMax over a non-column expression must be rejected"); @@ -2370,7 +2298,7 @@ async fn count_null_semantics_become_a_measure_filter() { "SELECT count(value) FROM samples", "SELECT count(value + 1) FROM samples", ] { - let qe = lower_sql(sql, &catalog, AccuracyTarget::exact()) + let qe = lower_sql(sql, &catalog) .await .unwrap_or_else(|error| panic!("{sql}: {error}")); assert!( @@ -2383,7 +2311,7 @@ async fn count_null_semantics_become_a_measure_filter() { "SELECT count(NULL) FROM samples", "SELECT count(nullable_value + 1) FROM samples", ] { - let qe = lower_sql(sql, &catalog, AccuracyTarget::exact()) + let qe = lower_sql(sql, &catalog) .await .unwrap_or_else(|error| panic!("{sql}: {error}")); let [Some(Predicate(cond))] = aggregate_filters(&qe) else { @@ -2398,7 +2326,6 @@ async fn count_null_semantics_become_a_measure_filter() { let qe = lower_sql( "SELECT count(*), count(nullable_value) FROM samples", &catalog, - AccuracyTarget::exact(), ) .await .unwrap(); @@ -2406,7 +2333,6 @@ async fn count_null_semantics_become_a_measure_filter() { let error = lower_sql( "SELECT count(nullable_value) FROM samples GROUP BY ROLLUP(value)", &catalog, - AccuracyTarget::exact(), ) .await .unwrap_err(); @@ -2435,7 +2361,6 @@ async fn grouped_map_column_preserves_map_type() { "SELECT labels, max(value) AS value FROM raw_samples GROUP BY labels ORDER BY labels", &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap(); @@ -2464,7 +2389,6 @@ async fn clickhouse_modulo_uses_native_arithmetic_types_and_nullability() { &format!("SELECT {call} AS value FROM numbers"), &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap(); @@ -2472,7 +2396,6 @@ async fn clickhouse_modulo_uses_native_arithmetic_types_and_nullability() { &format!("SELECT {native} AS value FROM numbers"), &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap(); @@ -2483,7 +2406,6 @@ async fn clickhouse_modulo_uses_native_arithmetic_types_and_nullability() { "SELECT modulo(n, 3) AS value FROM numbers", &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap() @@ -2519,14 +2441,9 @@ async fn original_o11y_map_queries_lower_with_typed_results() { include_str!("data/o11y_q09.sql"), include_str!("data/o11y_q12.sql"), ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap_or_else(|e| panic!("{sql}: {e}")); + let query = lower_sql_dialect(sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); let schema = &query.schema; assert!( schema @@ -2551,14 +2468,9 @@ async fn clickhouse_modulo_preserves_projection_names_and_outer_references() { "modulo(bytes, 3)", ), ] { - let query = lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let query = lower_sql_dialect(sql, &catalog(), SqlDialect::ClickhouseSQL) + .await + .unwrap(); assert_eq!(query.schema.fields[0].name, name); } } @@ -2585,7 +2497,6 @@ async fn clickhouse_map_access_keeps_generated_names_and_rejects_variant_coercio "SELECT labels['job'] FROM t", &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap(); @@ -2597,15 +2508,13 @@ async fn clickhouse_map_access_keeps_generated_names_and_rejects_variant_coercio "SELECT map()['a'] FROM t", &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .is_err()); assert!(lower_sql_dialect( "SELECT map('a', integer, 'b', floating) FROM t", &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact() + SqlDialect::ClickhouseSQL ) .await .is_err()); @@ -2633,14 +2542,9 @@ async fn arg_selector_result_schema_tracks_selected_argument() { true, ), ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let query = lower_sql_dialect(sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .unwrap(); let schema = &query.schema; assert_eq!(schema.fields[0].dtype, dtype); assert_eq!(schema.fields[0].nullable, nullable); @@ -2667,14 +2571,9 @@ async fn clickhouse_list_element_uses_canonical_typed_access() { ("SELECT arrayElement(samples, -1) AS selected FROM t", false), ("SELECT samples[index] AS selected FROM t", true), ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let query = lower_sql_dialect(sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .unwrap(); let output = &query.schema; assert_eq!(output.fields[0].dtype, DataType::Int64); assert_eq!(output.fields[0].nullable, nullable); @@ -2683,14 +2582,9 @@ async fn clickhouse_list_element_uses_canonical_typed_access() { } for sql in ["SELECT samples[0] FROM t", "SELECT samples['bad'] FROM t"] { assert!( - lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact() - ) - .await - .is_err(), + lower_sql_dialect(sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .is_err(), "{sql}" ); } @@ -2726,14 +2620,9 @@ async fn clickhouse_tuple_element_preserves_declared_field_metadata() { true, ), ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let query = lower_sql_dialect(sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .unwrap(); let output = &query.schema; assert_eq!(output.fields[0].dtype, dtype); assert_eq!(output.fields[0].nullable, nullable); @@ -2744,14 +2633,9 @@ async fn clickhouse_tuple_element_preserves_declared_field_metadata() { for selector in ["0", "-1", "3", "'missing'", "index"] { let sql = format!("SELECT tupleElement(sample, {selector}) FROM t"); assert!( - lower_sql_dialect( - &sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact() - ) - .await - .is_err(), + lower_sql_dialect(&sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .is_err(), "{sql}" ); } @@ -2778,13 +2662,9 @@ async fn composite_distinct_counts_tuples() { Field::plain("b", DataType::Int64, false), ]), ); - let composite = lower_sql( - "SELECT COUNT(DISTINCT a, b) FROM t", - &cat, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let composite = lower_sql("SELECT COUNT(DISTINCT a, b) FROM t", &cat) + .await + .unwrap(); let NonASAPOp::Aggregate { measures, .. } = op(find_aggregate_node(&composite).expect("expected an Aggregate")) else { @@ -2795,13 +2675,9 @@ async fn composite_distinct_counts_tuples() { "{measures:?}" ); - let single = lower_sql( - "SELECT COUNT(DISTINCT a) FROM t", - &cat, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let single = lower_sql("SELECT COUNT(DISTINCT a) FROM t", &cat) + .await + .unwrap(); let NonASAPOp::Aggregate { measures, .. } = op(find_aggregate_node(&single).expect("expected an Aggregate")) else { @@ -2824,13 +2700,9 @@ async fn composite_distinct_rejects_expression_arguments() { Field::plain("b", DataType::Int64, false), ]), ); - let error = lower_sql( - "SELECT COUNT(DISTINCT a, b + 1) FROM t", - &cat, - AccuracyTarget::exact(), - ) - .await - .unwrap_err(); + let error = lower_sql("SELECT COUNT(DISTINCT a, b + 1) FROM t", &cat) + .await + .unwrap_err(); assert!( matches!(&error, LoweringError::UnsupportedAggregate(reason) if reason.contains("non-column expression")), @@ -2853,7 +2725,7 @@ async fn distinct_with_derived_sibling() { "SELECT count(DISTINCT a, b), sum(b + 1) FROM t", "SELECT count(DISTINCT a, b), corr(a,b) FROM t", ] { - let result = lower_sql(sql, &catalog, AccuracyTarget::exact()).await; + let result = lower_sql(sql, &catalog).await; assert!(result.is_ok(), "{sql}: {result:?}"); } } @@ -2886,7 +2758,7 @@ async fn conditional_count_lowers_to_a_filtered_measure() { assert!( matches!( measures.as_slice(), - [AggIntent::Count { .. }, AggIntent::Sum { col: Some(3) }] + [AggIntent::Count, AggIntent::Sum { col: Some(3) }] ), "{measures:?}" ); @@ -2938,7 +2810,7 @@ async fn count_of_a_nullable_expression_filters_nulls() { assert!( matches!( find_aggregate(&qe).unwrap().1.as_slice(), - [AggIntent::Count { .. }] + [AggIntent::Count] ), "still a row count" ); @@ -2977,7 +2849,6 @@ async fn measure_filter_inside_a_rollup_is_rejected() { let err = lower_sql( "SELECT service, count(*) FILTER (WHERE latency > 1.0) FROM metrics GROUP BY ROLLUP(service)", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap_err(); @@ -2985,7 +2856,7 @@ async fn measure_filter_inside_a_rollup_is_rejected() { } async fn lower_err(sql: &str) -> String { - lower_sql(sql, &catalog(), AccuracyTarget::exact()) + lower_sql(sql, &catalog()) .await .err() .unwrap_or_else(|| panic!("expected {sql:?} to be rejected")) @@ -3137,7 +3008,6 @@ async fn duration_in_a_case_branch_is_rejected_as_a_duration() { let err = asap_frontend_sql::lower_sql( "SELECT CASE WHEN true THEN INTERVAL '1 day' ELSE d - d END FROM t", &catalog, - AccuracyTarget::exact(), ) .await .unwrap_err() @@ -3285,7 +3155,7 @@ async fn other_time_conditions_stay_scan_predicates() { WHERE ts >= now() - INTERVAL '5 minutes' AND ts < now() - INTERVAL '1 minute'", "SELECT COUNT(*) FROM events WHERE ts >= now() - INTERVAL '5 minutes'", ] { - let qe = lower_sql(sql, &catalog, AccuracyTarget::exact()) + let qe = lower_sql(sql, &catalog) .await .unwrap_or_else(|e| panic!("{sql}: {e}")); assert!(trailing_window(&qe).is_none(), "{sql}: {qe:#?}"); diff --git a/crates/frontend-sql/tests/temporal_types.rs b/crates/frontend-sql/tests/temporal_types.rs index e0380c31b..754812680 100644 --- a/crates/frontend-sql/tests/temporal_types.rs +++ b/crates/frontend-sql/tests/temporal_types.rs @@ -1,5 +1,4 @@ use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; fn catalog() -> SqlCatalog { SqlCatalog::new().with_table( @@ -13,13 +12,9 @@ async fn temporal_subtraction_rejects_unrepresentable_duration() { for dtype in [DataType::Date, DataType::Timestamp] { let catalog = SqlCatalog::new().with_table("t", Schema::new(vec![Field::plain("d", dtype, false)])); - let error = lower_sql( - "SELECT d - d AS elapsed FROM t", - &catalog, - AccuracyTarget::exact(), - ) - .await - .unwrap_err(); + let error = lower_sql("SELECT d - d AS elapsed FROM t", &catalog) + .await + .unwrap_err(); assert!( error.to_string().contains("unsupported duration type"), "{error}" @@ -33,9 +28,7 @@ async fn date_shifts_keep_their_type() { "SELECT d + INTERVAL '30' DAY AS shifted FROM t", "SELECT d - CAST('1 day' AS INTERVAL) AS shifted FROM t", ] { - let node = lower_sql(query, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let node = lower_sql(query, &catalog()).await.unwrap(); assert_eq!(node.schema.fields[0].dtype, DataType::Date); } } @@ -46,9 +39,7 @@ async fn interval_cast_lowers_like_interval_literal() { "SELECT INTERVAL '1' DAY AS duration FROM t", "SELECT CAST('1 day' AS INTERVAL) AS duration FROM t", ] { - let node = lower_sql(query, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let node = lower_sql(query, &catalog()).await.unwrap(); assert_eq!(node.schema.fields[0].dtype, DataType::Interval); } } @@ -62,7 +53,7 @@ async fn nested_temporal_subtraction_is_rejected() { "SELECT CASE WHEN true THEN INTERVAL '1 day' ELSE d - d END FROM t", "SELECT d FROM t ORDER BY d - d", ] { - let result = lower_sql(query, &catalog(), AccuracyTarget::exact()).await; + let result = lower_sql(query, &catalog()).await; let error = result.expect_err(&format!("accepted unsupported duration: {query}")); assert!( error.to_string().contains("unsupported duration type"), @@ -79,9 +70,7 @@ async fn negative_intervals_keep_their_type() { "SELECT -CAST('1 day' AS INTERVAL) AS duration FROM t", "SELECT -(INTERVAL '1 day' + INTERVAL '2 days') AS duration FROM t", ] { - let node = lower_sql(query, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let node = lower_sql(query, &catalog()).await.unwrap(); assert_eq!(node.schema.fields[0].dtype, DataType::Interval, "{query}"); } } @@ -93,9 +82,7 @@ async fn sql_date_literals_keep_their_type() { "SELECT DATE '2024-02-29' FROM t", "SELECT DATE '1969-12-31' + INTERVAL '1 day' FROM t", ] { - let node = lower_sql(query, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let node = lower_sql(query, &catalog()).await.unwrap(); assert_eq!(node.schema.fields[0].dtype, DataType::Date); } } diff --git a/crates/integration-tests/Cargo.toml b/crates/integration-tests/Cargo.toml index 055c031b1..c8ee40b02 100644 --- a/crates/integration-tests/Cargo.toml +++ b/crates/integration-tests/Cargo.toml @@ -10,22 +10,22 @@ asap-types = { path = "../types", features = ["test-support"] } asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["reference-pricer"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } asap-accuracy = { path = "../accuracy" } asap-cost = { path = "../cost" } -asap-plan-selection = { path = "../plan-selection", features = ["reference-pricer"] } +asap-plan-selection = { path = "../plan-selection" } # The fixtures plan through the facade and the developer tools' reference # enumeration. -asap-planner = { path = "../planner", features = ["test-support"] } +asap-planner = { path = "../planner" } asap-devtools = { path = "../devtools" } [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-accuracy = { path = "../accuracy", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } asap-devtools = { path = "../devtools" } # The tests plan PromQL and SQL workloads through the facade. -asap-planner = { path = "../planner", features = ["test-support"] } +asap-planner = { path = "../planner" } asap_sketchlib = { workspace = true } serde_json = "1" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 49534547c..a240bcb7c 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -13,7 +13,7 @@ pub mod fixtures { - use asap_types::ir::properties::AccuracyTarget; + use asap_types::accuracy::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::OperatorNode; @@ -95,58 +95,26 @@ pub mod fixtures { } } -/// Timing and physical ASAP DAG export helpers for planned candidates. +/// Physical ASAP DAG export helpers for planned candidates. pub mod physical_export { - use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPDAG}; - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - OperatorNode, - }; - use asap_types::physical::{Materialization, Medium}; + use asap_types::ir::OperatorNode; + use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use std::rc::Rc; - /// Time `root` under the default assignment (every sub-DAG - /// `NotMaterialized`, so every summary computed at query time). Returns the timed copy; read `node.timing` on it. - pub fn timed(root: &Rc) -> Rc { - timed_with(root, &MaterializationAssignment::default()) - } - - /// Time `root` with every sub-DAG `MatAtIngestion` where the DAG allows it. - pub fn at_ingestion(root: &Rc) -> Rc { - timed_with( - root, - &MaterializationAssignment::with_default(Materialization::MatAtIngestion( - Medium::Memory, - )), - ) - } - - fn timed_with( - root: &Rc, - assignment: &MaterializationAssignment, - ) -> Rc { - apply_materialization_timings(root, assignment, &mut NodeToTimedNodeCache::new()) - .expect("materialization timing failed") - } - - /// Time `root` (default assignment), then export the physical DAG. + /// Export the physical DAG of `root`. pub fn physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(&timed(root)).expect("physical ASAP DAG export failed") - } - - /// Time `root` with every sub-DAG `MatAtIngestion` where the DAG allows - /// it, then export the physical DAG. - pub fn at_ingestion_physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(&at_ingestion(root)).expect("physical ASAP DAG export failed") + asap_types::test_support::physical_asap_dag_at_query_time(root) + .expect("physical ASAP DAG export failed") } } /// Plans and candidates from the planner pipeline (the facade's -/// `run_planning`, with the developer tools' reference enumeration) for one +/// `run_planning`) for one /// query, with the built-in models. pub mod planning { - use asap_types::ir::properties::AccuracyTarget; + use asap_types::accuracy::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot}; + use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use asap_types::workload::{DataArrival, DataWorkload, SourceFacts}; use std::rc::Rc; @@ -179,13 +147,13 @@ pub mod planning { DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1_000.0, + max_out_of_order_ms: None, }, sources, } } - /// Every planning stage's output for the single query `root`, with - /// the reference enumeration of its memo. + /// Every planning stage's output for the single query `root`. pub fn run_all_stages( root: Rc, accuracy: AccuracyTarget, @@ -213,7 +181,6 @@ pub mod planning { range_evaluation: None, }], &data, - None, models, ) .expect("Pass 1 accepts the query") @@ -222,210 +189,75 @@ pub mod planning { /// The logical DAG the planner pipeline selects for the single query /// `root` at `accuracy`. pub fn selected_dag(root: Rc, accuracy: AccuracyTarget) -> Rc { - match run_all_stages(root, accuracy).run.selected_plan.as_deref() { - Some([QueryRoot::Operator(node)]) => Rc::clone(node), + match selected_operator_roots(&run_all_stages(root, accuracy)).as_slice() { + [node] => Rc::clone(node), other => panic!("one selected operator root: {other:?}"), } } - /// Exact computation and each explicit single-target rule application. - /// This fixture catalog is independent of the bounded MILP's visited plans. - pub fn explicit_local_candidates(root: Rc) -> Vec> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + /// `plan`'s cost per second (`asap_cost::plan_cost`) for queries with + /// `entries` over `data`, under `settings`' cost model 1 or 2. + pub fn plan_cost_per_second( + plan: &PhysicalASAPDAG, + entries: &[asap_types::workload::QueryWorkloadEntry], + data: &DataWorkload, + settings: &asap_cost::CostInputs, + ) -> f64 { + let recurrence: Vec<_> = entries + .iter() + .map(|entry| entry.recurrence.clone()) + .collect(); + let range_evaluations: Vec<_> = + entries.iter().map(|entry| entry.range_evaluation).collect(); + let (cost, _) = asap_cost::plan_cost( + plan, + &asap_cost::node_map(plan), + &recurrence, + &range_evaluations, + data, + settings, + 1.0, ) .unwrap(); - let mut choices = std::collections::BTreeSet::from([vec![0; local.targets.len()]]); - for (target, options) in local.targets.iter().enumerate() { - for choice in 1..options.choice_count() { - let mut selected = vec![0; local.targets.len()]; - selected[target] = choice; - choices.insert(selected); - } - } - choices + cost.cost_per_second.expect("cost model 1 or 2") + } + + /// The operator roots of the plan Stage 3 built in `run`, as IR. + fn selected_operator_roots( + run: &asap_devtools::planning_stages::PlanningStagesRun, + ) -> Vec> { + let (plan, _) = run.run.selection.as_ref().expect("a plan is selected"); + plan.query_roots() + .unwrap() .into_iter() - .map(|choice| { - match compose_logical_candidate(&local, &choice) - .expect("explicit local rule composes") - .remove(0) - { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator fixture"), - } + .filter_map(|root| match root { + QueryRoot::Operator(node) => Some(node), + QueryRoot::Scalar(_) => None, }) .collect() } - /// One explicit plan using `family` wherever a local target supports it, - /// keeping all other computations exact. No workload Cartesian enumeration. - pub fn explicit_summary_candidate( - root: Rc, - family: asap_types::ir::schema::SketchAlgorithm, - ) -> Rc { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let choices = local.targets.iter().map(|target| target.ways_to_compute.iter() - .position(|candidate| matches!(candidate, HowToComputeAggregateMeasure::Summary(found) if *found == family)).unwrap_or(0)).collect::>(); - match compose_logical_candidate(&local, &choices) - .expect("explicit family choices compose") - .remove(0) - { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator fixture"), - } - } - - /// Logical plans visited by the bounded search for `root`, in archive order. - /// This is not the generated candidate catalog. Rule-existence tests should - /// use explicit fixtures; failed compositions in this archive still panic. + /// The selected plan's operator roots for `root`. This is not the + /// generated candidate catalog: rule-existence tests should use explicit + /// fixtures. pub fn logical_candidates( root: Rc, accuracy: AccuracyTarget, ) -> Vec> { - require_composed_operator_candidates(run_all_stages(root, accuracy).logical_candidates()) + selected_operator_roots(&run_all_stages(root, accuracy)) } - /// [`logical_candidates`] with every output accepted whatever its - /// accuracy guarantee (`ignore_accuracy_guarantees`), so Pass 1 keeps - /// the summaries no query can use for `accuracy` - /// (`PruneReason::NoErrorBoundForQueryTarget`). For tests of a - /// rewrite's structure or execution over a summary whose output has no - /// guarantee that meets `accuracy` (e.g. a value filter over an - /// estimate), not of accuracy acceptance. + /// [`logical_candidates`] with accuracy not enforced + /// (`ignore_accuracy_guarantees`): no readout gets a required accuracy. + /// For tests of a rewrite's structure or execution over a summary whose + /// output has no guarantee that meets `accuracy` (e.g. a value filter + /// over an estimate), not of accuracy acceptance. pub fn logical_candidates_ignoring_guarantees( root: Rc, accuracy: AccuracyTarget, ) -> Vec> { let models = asap_planner::PlanningModels::builtin().with_ignore_accuracy_guarantees(true); - require_composed_operator_candidates( - run_all_stages_with(root, accuracy, models).logical_candidates(), - ) - } - - /// Fail on rejected composition rather than silently skipping a candidate. - pub fn require_composed_operator_candidates( - candidates: Vec, - ) -> Vec> { - candidates - .into_iter() - .filter_map(|candidate| { - match candidate - .roots - .unwrap_or_else(|e| panic!("{} does not compose: {e}", candidate.id)) - .remove(0) - { - QueryRoot::Operator(node) => Some(node), - QueryRoot::Scalar(_) => None, - } - }) - .collect() - } - /// Pass 1, local tumbling windows for `entries`' recurrences and every Pass 2 rule - /// on the seed grids (no query pruning). Rule tests read a group's - /// candidates by `OriginOfCandidateForMemoGroup`. - pub fn expanded_memo( - roots: Vec, - entries: &[asap_types::workload::QueryWorkloadEntry], - ) -> asap_types::stages::Memo { - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - use asap_logical_asap_optimization::test_support::{ - add_local_window_composition, try_populate_pass1, - }; - let initial = asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut memo = try_populate_pass1(initial, &inventory).unwrap(); - add_local_window_composition(&mut memo, entries); - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap() - } - - /// One plan of `memo`'s first `queries` queries (`memo::enumeration::build_logical_asap_dags_from_chosen_candidates`): - /// each query's first root group's DAG, every group `choice` does not - /// name implemented by its first candidate. - pub fn compose_memo_plan( - memo: &asap_types::stages::Memo, - queries: usize, - choice: &std::collections::BTreeMap, - ) -> Vec> { - use asap_logical_asap_optimization::build_logical_asap_dags_from_chosen_candidates; - use asap_types::stages::MemoGroupId; - let choice = choice - .iter() - .map(|(&group, &index)| (MemoGroupId(group as u32), index)) - .collect(); - let roots = build_logical_asap_dags_from_chosen_candidates(memo, &choice) - .expect("the choice composes"); - (0..queries) - .map(|query| { - let (_, root) = roots - .iter() - .find(|(group, _)| memo.query_roots[query].operator_refs().contains(&group)) - .expect("the query's root group"); - Rc::clone(root) - }) - .collect() - } - - /// Stage 2 and plan selection over fixed workload plans, without a timed - /// search: each `(name, roots)` is implemented by Stage 2 (plan ids start - /// with `name`), and all are priced together. - pub fn select_among( - variants: &[(&str, Vec)], - entries: &[asap_types::workload::QueryWorkloadEntry], - data: &DataWorkload, - ) -> ( - Vec, - asap_plan_selection::reference_pricer::WholePlanSelection, - ) { - let models = asap_planner::PlanningModels::builtin(); - let mut physical = Vec::new(); - for (name, roots) in variants { - // Composed nodes lose their guarantees; recompose them as - // `run_explicit_candidates` does. - let roots = asap_accuracy::propagation::recompose_guarantees(roots) - .unwrap_or_else(|error| panic!("{name} composes guarantees: {error}")); - // Composed outside the memo: sized as Pass 1 sizes a candidate. - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let roots = - asap_accuracy::sizing::with_summaries_sized_for_targets(&roots, &targets, data) - .unwrap_or_else(|error| panic!("{name} sizes its summaries: {error}")); - let mut plans = asap_physical_asap_optimization::optimize_physical_asap( - name, - &roots, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - ) - .unwrap_or_else(|error| panic!("Stage 2 implements {name}: {error}")); - for (index, plan) in plans.iter_mut().enumerate() { - plan.id = format!("{name}-{index}"); - } - physical.extend(plans); - } - let selection = - asap_devtools::memo_pricing::reference_select(&physical, entries, data, models) - .unwrap(); - (physical, selection) + selected_operator_roots(&run_all_stages_with(root, accuracy, models)) } /// Whether `node` builds a summary. @@ -448,12 +280,13 @@ pub mod planning { keep: impl Fn(&asap_types::stages::CandidateForMemoGroup) -> bool, ) -> Vec<(usize, usize)> { let mut found: Vec<_> = memo - .candidates + .groups() .iter() .enumerate() - .filter(|(group, _)| memo.groups[*group].reached_by_queries() == [q]) - .flat_map(|(group, candidates)| { - candidates + .filter(|(_, group)| group.reached_by_queries() == [q]) + .flat_map(|(group, memo_group)| { + memo_group + .candidates .iter() .enumerate() .filter(|(_, c)| keep(c)) @@ -463,13 +296,14 @@ pub mod planning { .collect(); // Group indices don't give the order: Pass 2 appends groups at the // end. Depth from the root does. - let mut depth = vec![usize::MAX; memo.groups.len()]; + let mut depth = vec![usize::MAX; memo.groups().len()]; let mut pending: std::collections::VecDeque<_> = memo - .groups + .groups() .iter() .enumerate() .filter(|(group, _)| { - memo.query_roots[q] + memo.queries[q] + .root .operator_refs() .contains(&&asap_types::stages::MemoGroupId(*group as u32)) }) @@ -481,9 +315,10 @@ pub mod planning { } depth[group] = d; pending.extend( - memo.groups[group] - .child_groups - .iter() + memo.groups()[group] + .expression + .children() + .into_iter() .map(|child| (child.0 as usize, d + 1)), ); } @@ -491,105 +326,6 @@ pub mod planning { found } - /// The selected plan's roots among fixed plans of `memo`'s queries, and - /// the selection's lowest prices of `separate` and of the `shared` plans: - /// - `separate{k}`: each query reads its own summary build, its topmost - /// own group's `k`-th Pass 1 candidate that builds one (the last when - /// it has fewer); - /// - `shared{g}`: every query reads the summary state group `g`, its own - /// estimate or a regrouping merge of it, when every query can. - pub fn select_rollup( - memo: &asap_types::stages::Memo, - queries: usize, - entries: &[asap_types::workload::QueryWorkloadEntry], - data: &DataWorkload, - ) -> (Vec>, f64, f64) { - let builds = |node: &Rc| OperatorNode::reachable(node).iter().any(is_build); - let own_builds: Vec> = (0..queries) - .map(|q| { - let found = own_choices(memo, q, |c| { - c.origin == asap_types::stages::OriginOfCandidateForMemoGroup::Pass1 - && builds(&c.root_node) - }); - let top = found - .first() - .unwrap_or_else(|| panic!("Q{} has a summary candidate", q + 1)) - .0; - found - .into_iter() - .filter(|(group, _)| *group == top) - .collect() - }) - .collect(); - let separate_plans = own_builds.iter().map(Vec::len).max().unwrap_or(0); - let mut plans: Vec<(String, Vec>)> = (0..separate_plans) - .map(|k| { - let choice = own_builds - .iter() - .map(|found| found[k.min(found.len() - 1)]) - .collect(); - ( - format!("separate{k}"), - compose_memo_plan(memo, queries, &choice), - ) - }) - .collect(); - for (state, group) in memo.groups.iter().enumerate() { - if !is_build(&group.logical_node) || group.reached_by_queries().len() < queries { - continue; - } - let reading: Option> = (0..queries) - .map(|q| { - own_choices(memo, q, |c| { - c.input_groups.iter().any(|child| child.0 as usize == state) - }) - .into_iter() - .next() - }) - .collect(); - if let Some(choice) = reading { - plans.push(( - format!("shared{state}"), - compose_memo_plan(memo, queries, &choice), - )); - } - } - assert!( - plans.len() > separate_plans, - "a plan in which every query reads one build" - ); - let named: Vec<(&str, Vec)> = plans - .iter() - .map(|(name, roots)| { - ( - name.as_str(), - roots.iter().cloned().map(QueryRoot::Operator).collect(), - ) - }) - .collect(); - let (physical, selection) = select_among(&named, entries, data); - let selected = physical - .iter() - .find(|plan| plan.id == selection.selected) - .expect("the selected plan"); - let roots = plans - .iter() - .find(|(name, _)| *name == selected.from_logical) - .map(|(_, roots)| roots.clone()) - .unwrap(); - let cheapest = |plans: &[(String, Vec>)]| { - plans - .iter() - .map(|(name, _)| cheapest_of(&selection, name)) - .fold(f64::INFINITY, f64::min) - }; - let separate = cheapest(&plans[..separate_plans]); - // Every separate plan rejected would make `shared < separate` hold - // trivially. - assert!(separate.is_finite(), "a valid plan with a build per query"); - (roots, separate, cheapest(&plans[separate_plans..])) - } - /// The summary builds and rollups (regrouping merges) `roots` read, each /// once. pub fn builds_and_rollups(roots: &[Rc]) -> (usize, usize) { @@ -609,18 +345,4 @@ pub mod planning { } (builds, rollups) } - - /// The lowest price per second among `selection`'s plans whose id starts - /// with `name`. - pub fn cheapest_of( - selection: &asap_plan_selection::reference_pricer::WholePlanSelection, - name: &str, - ) -> f64 { - selection - .costs - .iter() - .filter(|(id, _)| id.starts_with(&format!("{name}-"))) - .filter_map(|(_, cost)| cost.cost_per_second) - .fold(f64::INFINITY, f64::min) - } } diff --git a/crates/integration-tests/tests/aggregate.rs b/crates/integration-tests/tests/aggregate.rs index 8aa849b8c..0bbfaf354 100644 --- a/crates/integration-tests/tests/aggregate.rs +++ b/crates/integration-tests/tests/aggregate.rs @@ -13,8 +13,8 @@ use std::time::Duration; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::fixtures::metric_schema; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::{AggIntent, Reduction, Source}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; fn lower(q: &str) -> Rc { @@ -83,13 +83,7 @@ fn q06_sum_by_job() { fn q07_count_is_row_count() { assert_eq!( lower("count(http_requests_total)"), - agg( - vec![], - AggIntent::Count { - accuracy: AccuracyTarget::exact() - }, - scan("http_requests_total", &[]), - ), + agg(vec![], AggIntent::Count, scan("http_requests_total", &[]),), ); } @@ -184,11 +178,7 @@ fn q10_quantile_cross_series() { lower("quantile(0.99, http_requests_total)"), agg( vec![], - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::exact() - }, + AggIntent::Quantile { col: None, q: 0.99 }, scan("http_requests_total", &[]), ), ); diff --git a/crates/integration-tests/tests/binary_op.rs b/crates/integration-tests/tests/binary_op.rs index c787de998..59c453062 100644 --- a/crates/integration-tests/tests/binary_op.rs +++ b/crates/integration-tests/tests/binary_op.rs @@ -11,11 +11,11 @@ use std::time::Duration; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::fixtures::metric_schema; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::{ AggIntent, BinaryOpKind, GroupSide, Reduction, Source, VectorGrouping, VectorMatch, VectorMatchKind, }; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind}; use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode, ScalarExpr, TimeRangeKind}; diff --git a/crates/integration-tests/tests/declared_group_counts.rs b/crates/integration-tests/tests/declared_group_counts.rs deleted file mode 100644 index 6cad286f1..000000000 --- a/crates/integration-tests/tests/declared_group_counts.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! The groups plan selection prices a `by (...)` at, from each label's -//! declared distinct-value count (`SourceFacts::distinct_values_per_label`), -//! over the workload of #840: two per-series quantiles and two grouped -//! quantiles of one gauge with 10,000 series. - -use crate::planner_layering_common; - -use std::collections::BTreeSet; - -use asap_types::workload::PlanningWorkload; -use planner_layering_common::*; - -/// #840's workload (Q1–Q4) with `distinct_values_per_label` declared for -/// `data`. -fn workload(distinct_values_per_label: serde_json::Value) -> PlanningWorkload { - let query = |text: &str| { - serde_json::json!({ - "query": text, "demand": {"fixed_interval": 10000}, - "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, - "predictability": {"predictable": {"known_at": null}} - }) - }; - serde_json::from_value(serde_json::json!({ - "query_workload": {"language": "promql", "query_batch": null, "repeating_queries": [ - query("quantile_over_time(0.5, data[1m])"), - query("quantile_over_time(0.9, data[5m])"), - query("quantile by (host, job) (0.5, data)"), - query("quantile by (job) (0.5, data)"), - ]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "gauge", - "sample_interval": 15000, - "distinct_values_per_label": distinct_values_per_label}}} - })) - .expect("workload") -} - -/// Isolate the declared-cardinality pricing contract from search order. -fn priced_exact(workload: &PlanningWorkload) -> Run { - let roots = lower_promql(workload); - run_explicit_candidates( - workload, - roots.clone(), - vec![roots], - asap_planner::PlanningModels::builtin(), - ) -} - -/// The group counts every priced plan gives the builds and aggregates read -/// only by `query`, from their cost details (`... into N states`, `... into -/// N groups`). -fn priced_groups(run: &Run, query: usize) -> BTreeSet { - let mut groups = BTreeSet::new(); - for physical in &run.physical { - let Some(cost) = run.selection.costs.get(&physical.id) else { - continue; - }; - for node in &physical.dag().nodes { - if readers(physical.dag(), &physical.query_roots, node.id) != BTreeSet::from([query]) { - continue; - } - let detail = &cost.per_node[&node.id].detail; - let words: Vec<&str> = detail.split([' ', ':', ';']).collect(); - for window in words.windows(3) { - if window[0] == "into" && matches!(window[2], "states" | "groups") { - groups.insert(window[1].parse().expect("a group count")); - } - } - } - } - groups -} - -const Q3_BY_HOST_JOB: usize = 2; -const Q4_BY_JOB: usize = 3; - -/// With `host` and `job` declared, `by (host, job)` is priced at 1,000 × 10 -/// groups, at most the 10,000 series, and `by (job)` at 10. -#[test] -fn a_grouping_is_priced_at_the_product_of_its_labels_counts() { - let run = priced_exact(&workload(serde_json::json!({"host": 1000, "job": 10}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); - assert_eq!(priced_groups(&run, Q4_BY_JOB), BTreeSet::from([10])); -} - -/// A product above `series_count` is capped at it: each group holds at -/// least one series. -#[test] -fn a_grouping_has_at_most_one_group_per_series() { - let run = priced_exact(&workload(serde_json::json!({"host": 5000, "job": 10}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); -} - -/// A grouping with an undeclared label gets no invented count: each of the -/// 10,000 series is in one group, and each of the 10,000 samples the -/// instant selector reads in at most one, so `by (host, job)` with only -/// `host` declared, and both groupings with nothing declared, are priced at -/// 10,000 groups. -#[test] -fn a_grouping_with_an_undeclared_label_is_bound_by_the_series_read() { - let run = priced_exact(&workload(serde_json::json!({"host": 1000}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); - assert_eq!(priced_groups(&run, Q4_BY_JOB), BTreeSet::from([10_000])); - let run = priced_exact(&workload(serde_json::json!({}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); -} diff --git a/crates/integration-tests/tests/deployment_capabilities.rs b/crates/integration-tests/tests/deployment_capabilities.rs deleted file mode 100644 index 482ca9ac5..000000000 --- a/crates/integration-tests/tests/deployment_capabilities.rs +++ /dev/null @@ -1,271 +0,0 @@ -//! The deployment's capabilities (#509's execution-capability input) over -//! the planner-layering examples: the summaries the executor supports -//! restrict Stage 1 (Pass 1, and so Pass 2). - -use crate::planner_layering_common; - -use asap_cost::CostInputs; -use asap_devtools::planning_stages::lower_sql_roots; -use asap_logical_asap_optimization::{DeploymentCapabilities, SupportedSummary}; -use asap_types::ir::flat::FlatDag; -use asap_types::ir::schema::{SketchAlgorithm, StatisticKind}; -use asap_types::ir::{ASAPOp, Operator}; -use asap_types::physical::Materialization; -use planner_layering_common::*; - -/// A deployment whose executor supports only `supported`: each family with -/// its readouts. -fn supporting(supported: &[(SketchAlgorithm, &[StatisticKind])]) -> DeploymentCapabilities { - DeploymentCapabilities { - supported_summaries: Some( - supported - .iter() - .map(|(family, readouts)| SupportedSummary { - family: *family, - readouts: readouts.to_vec(), - }) - .collect(), - ), - ..DeploymentCapabilities::default() - } -} - -/// Every (family, readout) a logical candidate's DAG uses: each sketch -/// build with each statistic estimated from it. -fn summaries_used(dag: &FlatDag) -> Vec<(SketchAlgorithm, StatisticKind)> { - sketch_builds(dag) - .into_iter() - .flat_map(|(build, family, _)| { - estimates_of(dag, build) - .into_iter() - .map(move |(_, statistic)| (family, statistic.kind())) - }) - .collect() -} - -/// Capabilities filter each local computation before physical selection. -/// Compare explicit one-rule applications, not a timed search's archive. -#[test] -fn restricting_summaries_removes_exactly_the_candidates_that_use_them() { - let heap = supporting(&[(SketchAlgorithm::CountSketchWithHeap, &[StatisticKind::TopK])]); - assert_removes_exactly("example 1", lower_promql(&example1()), &heap); - let distinct_and_l2 = supporting(&[ - ( - SketchAlgorithm::UnivMon, - &[StatisticKind::Distinct, StatisticKind::FrequencyL2], - ), - (SketchAlgorithm::Hll, &[StatisticKind::Distinct]), - ]); - let (workload, catalog) = example2(); - assert_removes_exactly( - "example 2", - lower_sql_roots(&workload, &catalog).unwrap(), - &distinct_and_l2, - ); - let kll = supporting(&[(SketchAlgorithm::Kll, &[StatisticKind::Quantile])]); - assert_removes_exactly("pattern B", lower_promql(&pattern_b()), &kll); -} - -fn assert_removes_exactly( - name: &str, - roots: Vec, - deployment: &DeploymentCapabilities, -) { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let enumerate_local = - |inventory: &asap_logical_asap_optimization::test_support::PerComputationCandidates| { - let mut result = Vec::new(); - for (target, options) in inventory.targets.iter().enumerate() { - for choice in 0..options.choice_count() { - let mut choices = vec![0; inventory.targets.len()]; - choices[target] = choice; - if let Ok(roots) = compose_logical_candidate(inventory, &choices) { - result.push(asap_types::ir::flat::flatten(&roots).0); - } - } - } - result - }; - let unrestricted = enumerate_local(&local); - let restricted = enumerate_local( - &local.without_unsupported_summaries(deployment.supported_summaries.as_deref()), - ); - let expected = unrestricted - .iter() - .filter(|dag| { - summaries_used(dag) - .iter() - .all(|(family, readout)| deployment.supports(family, *readout)) - }) - .collect::>(); - assert!( - expected.len() < unrestricted.len(), - "{name}: unsupported candidates removed" - ); - assert!( - restricted.iter().any(|dag| !summaries_used(dag).is_empty()), - "{name}: supported summary retained" - ); - assert_eq!(restricted.iter().collect::>(), expected, "{name}"); -} - -/// Pattern B with a p50 panel beside its p99, both every minute over the -/// same 5-min window. One 1-min tumbling KLL chain serves both panels. -fn pattern_b_with_p50() -> asap_types::workload::PlanningWorkload { - let mut workload = pattern_b(); - let repeating = workload - .query_workload - .repeating_queries - .as_mut() - .expect("pattern B repeats"); - let mut p50 = repeating[0].clone(); - p50.query = asap_types::workload::Query("quantile_over_time(0.5, latency_ms[5m])".into()); - repeating.push(p50); - workload -} - -/// One explicitly shared KLL tumbling-window chain, passed through normal physical rules. -fn shared_tumbling_window_configurations(deployment: &DeploymentCapabilities) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::ir::QueryRoot; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = pattern_b_with_p50(); - let original = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&original), - ) - .unwrap(); - let choices = local - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - matches!( - candidate, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap() - }) - .collect::>(); - let composed = compose_logical_candidate(&local, &choices).unwrap(); - let shared = asap_logical_asap_optimization::test_support::share_common_sub_dags( - composed - .into_iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, node), - _ => unreachable!(), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect::>(); - let tumbling_windows = tumbling_window_candidates(&shared, &[Some(60_000), Some(60_000)]); - assert_eq!(tumbling_windows.len(), 1, "both estimators share one build"); - let tumbling_window_roots = window_composition_pass2_candidates(&shared, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - let settings = CostInputs::default(); - run_explicit_candidates( - &workload, - original, - vec![tumbling_window_roots], - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(deployment), - ) -} - -/// Physical candidate ids with their cost per second. -type CandidateCosts = Vec<(String, f64)>; - -/// The per-second cost of every physical candidate whose tumbling-window chain both -/// panels read is materialized at query time: shared within one evaluation -/// (`MatAtQuery`), and kept across evaluations -/// (`MatAtQueryKeptAcrossEvaluations`, which Stage 3 prices as such). -fn shared_chain_costs(run: &Run) -> (CandidateCosts, CandidateCosts) { - let (mut within_one, mut kept) = (Vec::new(), Vec::new()); - for p in &run.physical { - for n in &p.dag().nodes { - let shared_chain = matches!( - n.payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && readers(p.dag(), &p.query_roots, n.id).len() == 2; - let Some(cost) = run.selection.costs.get(&p.id).filter(|_| shared_chain) else { - continue; - }; - let entry = (p.id.clone(), cost.cost_per_second.expect("cost model 1")); - let kept_detail = cost.per_node[&n.id].detail.contains("across evaluations"); - match n.materialization { - Materialization::MatAtQuery(_) => within_one.push(entry), - Materialization::MatAtQueryKeptAcrossEvaluations(_) if kept_detail => { - kept.push(entry) - } - Materialization::MatAtQueryKeptAcrossEvaluations(_) => { - panic!("{}: a kept chain priced as not kept", p.id) - } - _ => {} - } - } - } - (within_one, kept) -} - -/// With query-time results retained, a chain two panels read is offered -/// both ways: shared within one evaluation (rebuilt from the 5-min window -/// every minute, then reused by both panels) and kept across evaluations -/// (B3: one new tumbling window per evaluation, the 4 older tumbling windows held in memory -/// between evaluations). Holding 1M per-series KLLs between evaluations -/// costs far more than rebuilding them, so the shared option is cheaper -/// and the kept one is not selected. Without query-time retention, the -/// kept option is not offered. -#[test] -fn query_time_results_retained_still_offers_the_chain_shared_within_one_evaluation() { - let keeping = DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - }; - let run = shared_tumbling_window_configurations(&keeping); - let (within_one, kept) = shared_chain_costs(&run); - assert!(!within_one.is_empty(), "no shared option: kept {kept:?}"); - assert!(!kept.is_empty(), "no kept option: shared {within_one:?}"); - let cheapest = |options: &[(String, f64)]| { - options - .iter() - .map(|(_, cost)| *cost) - .fold(f64::INFINITY, f64::min) - }; - assert!( - cheapest(&within_one) < cheapest(&kept), - "shared {within_one:?}, kept {kept:?}" - ); - assert!( - kept.iter().all(|(id, _)| *id != run.selection.selected), - "{}", - run.selection.selected - ); - assert_selects_cheapest_valid(&run); - - let (_, kept) = shared_chain_costs(&shared_tumbling_window_configurations( - &DeploymentCapabilities::default(), - )); - assert!(kept.is_empty(), "{kept:?}"); -} diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs deleted file mode 100644 index 63d051298..000000000 --- a/crates/integration-tests/tests/exact_composition.rs +++ /dev/null @@ -1,332 +0,0 @@ -//! Issue #171 — composing exact operators with summary plans across -//! explicit update/evaluation boundaries, through the planner stages' -//! logical candidates (Stage 1). -//! -//! Covers grouped fine-to-coarse and identity folds over a summary -//! evaluation, phase-aware summary construction and pre/post-ASAP schemas, -//! along with pins for every exact-accumulator nesting under an outer -//! sketch. - -use std::rc::Rc; - -use asap_integration_tests::physical_export::{at_ingestion, physical_asap_dag, timed}; -use asap_integration_tests::planning::explicit_local_candidates; -use asap_types::ir::operator::agg_intent::{default_quantile, AggIntent}; -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{ExactKind, FieldDataType, SummaryUpdate}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; - -use asap_types::ir::properties::AccuracyTarget; - -// ── fixtures ──────────────────────────────────────────────────────────── - -fn node(op: NonASAPOp) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(op)) - .expect("fixture node derives its schema") -} - -fn metric_scan(labels: &[&str]) -> Rc { - let mut columns = vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ]; - columns.extend( - labels - .iter() - .map(|n| Field::plain(*n, DataType::Utf8, true)), - ); - node(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency".into(), - }, - predicates: vec![], - schema: Schema::with_time_index(columns, 0, vec![]), - }) -} - -fn agg(by: Vec, intent: AggIntent, child: Rc) -> Rc { - node(NonASAPOp::Aggregate { - reduction: Reduction::by(by), - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child, - }) -} - -fn per_entity(intent: AggIntent, child: Rc) -> Rc { - node(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child, - }) -} - -/// `quantile by (zone, host) (latency)` — the fine-grained inner summary. -fn fine_quantile() -> Rc { - agg( - vec![2, 3], - default_quantile(0.99), - metric_scan(&["zone", "host"]), - ) -} - -/// The Stage 1 logical candidate of the single query `root` (planned at -/// ε = 0.01) whose outer aggregate stays exact over the inner summary -/// evaluation: the read-time composition. -fn composed(root: &Rc) -> Rc { - explicit_local_candidates(Rc::clone(root)) - .into_iter() - .find(|candidate| { - !matches!( - below_projects(candidate).asap(), - Some(ASAPOp::SummaryEstimate { .. }) - ) && summary_evaluation(candidate).is_some() - }) - .expect("an exact fold over a summary evaluation") -} - -/// `node` with every `Project` above it skipped: Stage 1 restores a -/// rewritten aggregate's output names with a `Project`. -fn below_projects(mut node: &Rc) -> &Rc { - while let Some(NonASAPOp::Project { child, .. }) = node.non_asap() { - node = child; - } - node -} - -/// The first summary evaluation in `node`'s DAG. -fn summary_evaluation(node: &Rc) -> Option<&Rc> { - if matches!(node.asap(), Some(ASAPOp::SummaryEstimate { .. })) { - return Some(node); - } - node.children().into_iter().find_map(summary_evaluation) -} - -fn is_plain(node: &OperatorNode) -> bool { - node.schema - .fields - .iter() - .all(|f| matches!(f.dtype, FieldDataType::Plain(_))) -} - -fn names(node: &OperatorNode) -> Vec<&str> { - node.schema.fields.iter().map(|f| f.name.as_str()).collect() -} - -// ── step 1: pin every already-supported exact-accumulator nesting ─────── - -#[test] -fn every_exact_accumulator_is_finalized_before_an_outer_sketch() { - use std::time::Duration; - let cases: Vec<(Rc, ExactKind)> = vec![ - ( - agg( - vec![2], - AggIntent::Sum { col: None }, - metric_scan(&["zone"]), - ), - ExactKind::Sum, - ), - ( - agg( - vec![2], - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, - metric_scan(&["zone"]), - ), - ExactKind::Count, - ), - ( - agg( - vec![2], - AggIntent::Min { col: None }, - metric_scan(&["zone"]), - ), - ExactKind::Min, - ), - ( - agg( - vec![2], - AggIntent::Max { col: None }, - metric_scan(&["zone"]), - ), - ExactKind::Max, - ), - ( - per_entity( - AggIntent::Rate, - node(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: metric_scan(&["zone"]), - }), - ), - ExactKind::Rate, - ), - ( - per_entity( - AggIntent::Increase, - node(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: metric_scan(&["zone"]), - }), - ), - ExactKind::Increase, - ), - ]; - for (inner, kind) in cases { - let outer = agg(vec![], default_quantile(0.9), inner); - // The KLL candidate over the inner exact accumulator. - let root = explicit_local_candidates(Rc::clone(&outer)) - .into_iter() - .map(|candidate| Rc::clone(below_projects(&candidate))) - .find(|candidate| { - matches!(candidate.asap(), Some(ASAPOp::SummaryEstimate { summary_input, .. }) - if matches!(summary_input.asap(), Some(ASAPOp::SummaryAgg { child, .. }) - if matches!(child.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))) - }) - .unwrap_or_else(|| panic!("{kind:?}: a KLL over the exact accumulator")); - // Timing is not stored on the plan: time it with the outer summary - // maintained (which also validates every edge) and inspect the copy. - let root = at_ingestion(&root); - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { - panic!("expected KLL evaluation, got {:?}", root.operator); - }; - let Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) = &summary_input.operator else { - panic!("expected outer SummaryAgg"); - }; - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: finalized }) = &child.operator - else { - panic!("{kind:?}: missing maintenance finalization"); - }; - assert_eq!( - child.timing, - Some(ExecutionTiming::IngestionTime), - "{kind:?}: finalization runs at maintenance time" - ); - assert!( - matches!( - &finalized.operator, - Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::ExactAggregate(k, _), .. }) if *k == kind - ), - "{kind:?}: expected the exact accumulator under its finalization, got {:?}", - finalized.operator - ); - } -} - -// ── direction 1: outer exact fold over an inner summary evaluation ──────── - -/// `max`/`avg` over a quantile composes as an exact fold at query time over -/// the inner quantile's evaluation, with the pre-ASAP target's own schema. -#[test] -fn max_and_avg_over_quantile_compose_at_query_time() { - for intent in [AggIntent::Max { col: None }, AggIntent::Avg { col: None }] { - let root = agg(vec![0], intent.clone(), fine_quantile()); - let composed = composed(&root); - let child = summary_evaluation(&composed).unwrap(); - assert!( - child.guarantee.is_some(), - "child has its KLL rank guarantee" - ); - assert!( - composed.guarantee.is_none(), - "rank error has no definition-backed conversion through max/average" - ); - assert!(is_plain(&composed)); - assert_eq!( - names(&composed), - root.schema - .fields - .iter() - .map(|c| c.name.as_str()) - .collect::>(), - "the composed plan's schema is the pre-ASAP target's own" - ); - assert_eq!( - timed(&composed).timing, - Some(ExecutionTiming::QueryTime), - "{intent:?}: the exact fold runs at query time" - ); - } -} - -/// Grouped fine-to-coarse fold (`by (zone)` over `by (zone, host)`) and the -/// identity fold (`by (zone)` over `by (zone)`) both compose; the operator -/// is the same, only the fold's row multiplicity differs. -#[test] -fn identity_and_genuine_multi_row_folds_both_compose() { - let identity_inner = agg(vec![2], default_quantile(0.99), metric_scan(&["zone"])); - for (label, inner) in [ - ("identity", identity_inner), - ("fine-to-coarse", fine_quantile()), - ] { - let root = agg(vec![0], AggIntent::Max { col: None }, inner); - let composed = composed(&root); - assert_eq!( - timed(&composed).timing, - Some(ExecutionTiming::QueryTime), - "{label}" - ); - assert_eq!(names(&composed), vec!["zone", "max"], "{label}"); - } -} - -// ── summary construction phase ────────────────────────────────────────── - -/// Summary construction can consume query-time values without pretending -/// they are available to an ingestion-time consumer. -#[test] -fn summary_construction_follows_its_value_input_phase() { - let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); - let post = composed(&root); - let illegal = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: post, - family: FieldDataType::ExactAggregate( - ExactKind::Max, - asap_types::ir::schema::ExactParams::Max, - ), - input: SummaryUpdate::column(asap_types::ir::scalar::ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - }), - Schema::lifted(vec![], None), - ) - .with_guarantee(None), - ); - // Even under an ingestion-time consumer the state is built at query - // time, because a evaluation sits below it. - let state = asap_types::ir::planned_data_state(&illegal, ExecutionTiming::IngestionTime); - assert_eq!(state.timing, ExecutionTiming::QueryTime); - asap_types::ir::validate_at_ingestion(&illegal, state.timing).unwrap(); -} - -// ── Physical export: explicit timing ───────────────────────────────────── - -#[test] -fn physical_export_carries_explicit_timing_for_a_composed_plan() { - let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); - let composed = composed(&root); - // Timing is explicit in the wire-6 DAG: the root finalizes the exact - // `max` state over the quantile evaluation, at query time. - let wire = physical_asap_dag(&composed); - let wire_root = wire.nodes.iter().find(|n| n.id == wire.roots[0]).unwrap(); - assert!(matches!( - wire_root.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) - )); - assert_eq!(wire_root.output_state.timing, ExecutionTiming::QueryTime); -} diff --git a/crates/integration-tests/tests/filtered_aggregates.rs b/crates/integration-tests/tests/filtered_aggregates.rs index 47dfd4efd..c95b9d7f8 100644 --- a/crates/integration-tests/tests/filtered_aggregates.rs +++ b/crates/integration-tests/tests/filtered_aggregates.rs @@ -1,7 +1,6 @@ //! Filtered aggregates (`FILTER (WHERE …)`, a per-measure filter, or a //! filtered `SummaryAgg`) execute with SQL semantics: a row that fails the //! filter contributes nothing, but its group is kept. -use crate::physical_common; use std::rc::Rc; use std::sync::Arc; @@ -14,11 +13,12 @@ use asap_executor::{ }; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::planning::{logical_candidates_ignoring_guarantees, selected_dag}; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::schema::FieldDataType; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{ExactKind, FieldDataType, SketchAlgorithm}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate}; +use asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; +use asap_types::summary_type::{ExactKind, SketchAlgorithm}; use futures::StreamExt; fn catalog() -> SqlCatalog { @@ -57,7 +57,7 @@ fn rows() -> Vec> { /// Run the logical DAG `root` over [`rows`]; rows come back shown and sorted. fn execute(root: &Rc) -> Vec { - let dag = physical_common::compile_physical_asap_dag(root).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); let scan = dag .nodes .iter() @@ -113,16 +113,13 @@ fn execute(root: &Rc) -> Vec { out } -async fn lower(sql: &str, accuracy: AccuracyTarget) -> Rc { - lower_sql(sql, &catalog(), accuracy).await.unwrap() +async fn lower(sql: &str) -> Rc { + lower_sql(sql, &catalog()).await.unwrap() } /// The selected plan for an exact `sql`, through all three stages. async fn exact_plan(sql: &str) -> Rc { - selected_dag( - lower(sql, AccuracyTarget::exact()).await, - AccuracyTarget::exact(), - ) + selected_dag(lower(sql).await, AccuracyTarget::exact()) } /// `COUNT(*) FILTER (WHERE x > 0) GROUP BY g` counts the matching rows per @@ -155,20 +152,18 @@ async fn exact_count_of_nullable_column_skips_nulls() { fn with_summary_filter(root: &Rc, filter: &Predicate) -> Rc { if let Operator::ASAP(ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, reduction, - grouping, filter: None, }) = &root.operator { return Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: child.clone(), - family: family.clone(), + summary_type: family.clone(), input: input.clone(), reduction: reduction.clone(), - grouping: grouping.clone(), filter: Some(filter.clone()), })) .unwrap(), @@ -182,7 +177,7 @@ fn with_summary_filter(root: &Rc, filter: &Predicate) -> Rc (Predicate, Schema) { - OperatorNode::reachable(&lower(sql, AccuracyTarget::exact()).await) + OperatorNode::reachable(&lower(sql).await) .into_iter() .find_map(|n| match n.non_asap() { Some(NonASAPOp::Aggregate { filters, child, .. }) => { @@ -205,7 +200,7 @@ async fn hand_filtered( ) -> Vec> { let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); let mut result = vec![]; - for root in logical_candidates_ignoring_guarantees(lower(sql, target.clone()).await, target) { + for root in logical_candidates_ignoring_guarantees(lower(sql).await, target) { let builds: Vec<_> = OperatorNode::reachable(&root) .into_iter() .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) @@ -214,7 +209,7 @@ async fn hand_filtered( continue; }; let Some(ASAPOp::SummaryAgg { - family: actual, + summary_type: actual, child, .. }) = build.asap() @@ -259,7 +254,7 @@ async fn hand_filtered_kll_compiles_and_executes() { .await; let roots = hand_filtered( "SELECT g, approx_percentile_cont(x, 0.5) AS q FROM events GROUP BY g", - |f| matches!(f, FieldDataType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::Kll), + |f| matches!(f, FieldDataType::Sketch(kind) if kind == &SketchAlgorithm::Kll), &filter, ) .await; @@ -273,15 +268,16 @@ async fn hand_filtered_kll_compiles_and_executes() { /// [`hand_filtered`]). async fn kll_candidates(sql: &str) -> Vec> { let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let roots: Vec<_> = logical_candidates_ignoring_guarantees(lower(sql, target.clone()).await, target) - .into_iter() - .filter(|root| { - OperatorNode::reachable(root).iter().any(|n| { - matches!(n.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::Kll) - }) + let roots: Vec<_> = + logical_candidates_ignoring_guarantees(lower(sql).await, target) + .into_iter() + .filter(|root| { + OperatorNode::reachable(root).iter().any(|n| { + matches!(n.asap(), Some(ASAPOp::SummaryAgg { summary_type: FieldDataType::Sketch(kind), .. }) + if kind == &SketchAlgorithm::Kll) }) - .collect(); + }) + .collect(); assert!(!roots.is_empty(), "no KLL candidate for {sql}"); roots } diff --git a/crates/integration-tests/tests/finer_aggregate_reuse.rs b/crates/integration-tests/tests/finer_aggregate_reuse.rs index ed8a1b06e..9ce0534a7 100644 --- a/crates/integration-tests/tests/finer_aggregate_reuse.rs +++ b/crates/integration-tests/tests/finer_aggregate_reuse.rs @@ -2,18 +2,10 @@ //! when its groups are coarser, a rollup of values or states (#950). use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; use asap_devtools::planning_stages::run_search_over; -use asap_executor::values::Value; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; + use serde_json::json; use std::path::Path; -use crate::physical_common; - const P90: &str = "percentile_cont(0.9) WITHIN GROUP (ORDER BY value)"; const WINDOW: &str = "ts > now() - INTERVAL '5' MINUTE"; @@ -41,257 +33,6 @@ fn workload(queries: &[String]) -> FuzzWorkload { w } -/// The expanded candidate memo of `queries`, as plan selection receives it. -fn expanded(queries: &[String]) -> Memo { - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(queries)) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap() -} - -/// The aggregate group of query `query` alone and its rollup candidates. -fn rollups(memo: &Memo, query: usize) -> (usize, Vec<&CandidateForMemoGroup>) { - let owner = memo - .groups - .iter() - .position(|group| { - group.reached_by_queries() == [query] - && matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) - }) - .expect("the query's own aggregate group"); - let candidates = memo.candidates[owner] - .iter() - .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) - .collect(); - (owner, candidates) -} - -/// Whether `candidate` reads another group's result through a `Filter` -/// (the value path), rather than merging summary states. -fn reads_finer_result(candidate: &CandidateForMemoGroup) -> bool { - !OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. }))) - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!(node.non_asap(), Some(NonASAPOp::Filter { .. }))) -} - -fn row(city: Option<&str>, host: &str, value: Option) -> Vec { - vec![ - Value::Timestamp(0), - city.map_or(Value::Null, |city| Value::Utf8(city.into())), - Value::Utf8(host.into()), - value.map_or(Value::Null, Value::Float64), - ] -} - -/// Row sets covering an empty input, an absent selected group, NULL samples, -/// several hosts per city and other cities. -fn row_sets() -> Vec>> { - vec![ - vec![], - vec![row(Some("LA"), "h1", Some(5.0))], - vec![row(Some("NYC"), "h1", None)], - vec![ - row(Some("NYC"), "h1", Some(10.0)), - row(Some("NYC"), "h1", Some(30.0)), - row(Some("NYC"), "h2", Some(20.0)), - row(Some("SF"), "h1", Some(7.0)), - row(Some("LA"), "h1", Some(90.0)), - row(None, "h1", Some(1.0)), - ], - ] -} - -/// The candidate gives the original query's rows on every row set. -fn assert_same_rows(memo: &Memo, owner: usize, candidate: &CandidateForMemoGroup) { - let sorted = |rows: Vec>| { - let mut rows: Vec<_> = rows.iter().map(|row| format!("{row:?}")).collect(); - rows.sort(); - rows - }; - for rows in row_sets() { - let before = physical_common::execute_raw_rows_at( - &memo.groups[owner].logical_node, - rows.clone(), - 1000, - ); - let after = physical_common::execute_raw_rows_at(&candidate.root_node, rows.clone(), 1000); - assert_eq!(sorted(after), sorted(before), "rows {rows:?}"); - } -} - -/// Example A: both dropped keys are pinned, so the quantile query reads one -/// row of the finer quantile result; no quantile rollup is needed. -#[test] -fn pinned_keys_read_one_finer_quantile_row() { - let memo = expanded(&[ - format!("SELECT city, host, {P90} FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT {P90} FROM metrics WHERE {WINDOW} AND city = 'NYC' AND host = 'h1'"), - ]); - let (owner, candidates) = rollups(&memo, 1); - let read = candidates - .into_iter() - .find(|c| reads_finer_result(c)) - .expect("a read of the finer quantile result"); - assert_same_rows(&memo, owner, read); -} - -/// Example B: a filtered global sum keeps the `city = 'NYC'` rows of the -/// finer sums and sums them. -#[test] -fn filtered_sum_rolls_up_finer_sums() { - let memo = expanded(&[ - format!("SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT SUM(value) FROM metrics WHERE {WINDOW} AND city = 'NYC'"), - ]); - let (owner, candidates) = rollups(&memo, 1); - let read = candidates - .into_iter() - .find(|c| reads_finer_result(c)) - .expect("a rollup of the finer sums"); - assert_same_rows(&memo, owner, read); -} - -/// A grouped query whose filter keeps several groups: filter, then roll the -/// finer `(city, host)` sums and maxima up to `city`. -#[test] -fn filtered_grouped_aggregates_roll_up_finer_values() { - let memo = expanded(&[ - format!( - "SELECT city, host, SUM(value), MAX(value) FROM metrics WHERE {WINDOW} GROUP BY city, host" - ), - format!( - "SELECT city, SUM(value), MAX(value) FROM metrics WHERE {WINDOW} AND city IN ('NYC', 'SF') GROUP BY city" - ), - ]); - let (owner, candidates) = rollups(&memo, 1); - let read = candidates - .into_iter() - .find(|c| reads_finer_result(c)) - .expect("a filtered rollup of the finer values"); - assert_same_rows(&memo, owner, read); -} - -/// Example C: two cities' p90 values cannot be combined, but their KLL -/// states can: keep the NYC and SF states, merge them, estimate. The merged -/// read gives the coarse KLL read's rows, SQL's one NULL row over an empty -/// selection included; the query's own read keeps the rows the `OR` form -/// keeps. -#[test] -fn filtered_kll_states_merge() { - let memo = expanded(&[ - format!("SELECT city, {P90} FROM metrics WHERE {WINDOW} GROUP BY city"), - format!("SELECT {P90} FROM metrics WHERE {WINDOW} AND city IN ('NYC', 'SF')"), - ]); - let (owner, candidates) = rollups(&memo, 1); - let merged = candidates - .into_iter() - .find(|candidate| { - OperatorNode::reachable(&candidate.root_node).iter().any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryMerge { children, group_by: Some(_), .. }) - if matches!(children[0].non_asap(), Some(NonASAPOp::Filter { .. }))) - }) - }) - .expect("a candidate merging the filtered finer states"); - let own_kll = |memo: &Memo, owner: usize| { - memo.candidates[owner] - .iter() - .find(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::Pass1 - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family, .. }) - if format!("{family:?}").contains("Kll")) - }) - }) - .expect("the query's own KLL candidate") - .clone() - }; - let direct = own_kll(&memo, owner); - let or_memo = expanded(&[format!( - "SELECT {P90} FROM metrics WHERE {WINDOW} AND (city = 'NYC' OR city = 'SF')" - )]); - let or_direct = own_kll(&or_memo, rollups(&or_memo, 0).0); - for rows in row_sets() { - let before = physical_common::execute_raw_rows_at(&direct.root_node, rows.clone(), 1000); - let or_form = - physical_common::execute_raw_rows_at(&or_direct.root_node, rows.clone(), 1000); - assert_eq!( - format!("{before:?}"), - format!("{or_form:?}"), - "rows {rows:?}" - ); - let after = physical_common::execute_raw_rows_at(&merged.root_node, rows.clone(), 1000); - assert_eq!(format!("{after:?}"), format!("{before:?}"), "rows {rows:?}"); - assert_eq!(after.len(), 1); - } -} - -/// The same groups in another key order are one computation: neither -/// aggregate reads the other (two such reads would be a cycle). -#[test] -fn permuted_group_by_is_not_a_finer_read() { - let memo = expanded(&[ - format!("SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT host, city, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY host, city"), - ]); - for query in 0..2 { - let (_, candidates) = rollups(&memo, query); - assert!(!candidates.into_iter().any(reads_finer_result)); - } -} - -/// A conjunct on a non-key column cannot be applied to finer sums. -#[test] -fn value_conjunct_has_no_finer_read() { - let memo = expanded(&[ - format!("SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT SUM(value) FROM metrics WHERE {WINDOW} AND city = 'NYC' AND value > 3"), - ]); - let (_, candidates) = rollups(&memo, 1); - assert!(!candidates.into_iter().any(reads_finer_result)); -} - -/// A dropped key that is not pinned leaves several quantile values per -/// group, and quantile values do not roll up. -#[test] -fn unpinned_key_has_no_quantile_value_read() { - let memo = expanded(&[ - format!("SELECT city, host, {P90} FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT {P90} FROM metrics WHERE {WINDOW} AND city = 'NYC'"), - ]); - let (_, candidates) = rollups(&memo, 1); - assert!(!candidates.into_iter().any(reads_finer_result)); -} - -/// The finer result lacks rows the query needs: compensation only removes rows. -#[test] -fn narrower_finer_aggregate_has_no_read() { - let memo = expanded(&[ - format!( - "SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} AND city = 'SF' GROUP BY city, host" - ), - format!("SELECT SUM(value) FROM metrics WHERE {WINDOW} AND city = 'NYC'"), - ]); - let (_, candidates) = rollups(&memo, 1); - assert!(!candidates.into_iter().any(reads_finer_result)); -} - /// The selected plan's cost per second for `queries`. fn cost(queries: &[String]) -> f64 { use asap_cost::CostInputs; @@ -311,11 +52,11 @@ fn cost(queries: &[String]) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); - let selection = run.run.selection.as_ref().unwrap(); - let value = selection.cost.cost_per_second.unwrap(); + let (plan, _) = run.run.selection.as_ref().unwrap(); + let value = + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings); println!("cost={value}"); value } diff --git a/crates/integration-tests/tests/frequency_execution.rs b/crates/integration-tests/tests/frequency_execution.rs deleted file mode 100644 index 24c083091..000000000 --- a/crates/integration-tests/tests/frequency_execution.rs +++ /dev/null @@ -1,148 +0,0 @@ -//! #509 Example 2's PromQL stand-in executes natively: the distinct count, -//! entropy and L2 norm of one series' sample values over 1 min, through the -//! exact candidate and through the one that builds a UnivMon, at query time. - -use std::collections::BTreeMap; -use std::rc::Rc; - -use asap_executor::physical_planner::{compile, promql_rows}; -use asap_executor::runtime::Scope; -use asap_executor::values::Value; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::explicit_local_candidates; -use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOperatorPayload}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, OperatorNode, -}; - -use crate::physical_common; - -const EVALUATION_MS: i64 = 300_000; -/// `flows_src_ip` samples: source 1 twice, 2 three times, 3 once. -const SAMPLES: [f64; 6] = [1.0, 2.0, 1.0, 2.0, 3.0, 2.0]; - -/// The summary family a candidate builds: `None` for the exact plan. -fn built_family(candidate: &Rc) -> Option { - OperatorNode::reachable(candidate) - .iter() - .find_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => Some(family.clone()), - _ => None, - }) -} - -/// Execute `candidate` at query time over `SAMPLES`, all inside the window. -fn execute(candidate: &Rc) -> Vec { - let timed = apply_materialization_timings( - candidate, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - let ranges = dag - .nodes - .iter() - .filter(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::TimeRange { .. }) - ) - }) - .count(); - assert_eq!(ranges, 1, "one range selector"); - // The whole plan runs at query time over the raw samples. - let compiled = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - let labels = BTreeMap::from([("__name__".to_string(), "flows_src_ip".to_string())]); - let store = physical_common::raw_data_store(&dag, |_, schema| { - SAMPLES - .iter() - .enumerate() - .map(|(i, value)| { - promql_rows::series_row(schema, &labels, 290_000 + i as i64, *value).unwrap() - }) - .collect() - }); - let result = physical_common::execute( - &compiled, - store, - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - ); - result[0] - .iter() - .flat_map(|batch| batch.to_rows()) - .flatten() - .filter_map(|value| match value { - Value::Float64(value) => Some(value), - Value::Int64(value) => Some(value as f64), - _ => None, - }) - .collect() -} - -// The exact candidate and the UnivMon candidate of each statistic both -// execute and give the statistic of the window's sample values (entropy in -// bits). -#[test] -fn standin_exact_and_univmon_candidates_execute() { - let entropy = -[2.0_f64, 3.0, 1.0] - .iter() - .map(|count| count / 6.0 * (count / 6.0).log2()) - .sum::(); - // A UnivMon sized for a distinct count at ε = 0.01 holds a heap of - // millions of keys on each of its 19 layers, past the default 64 MiB. - for (query, expected, epsilon, delta) in [ - ("distinct_over_time(flows_src_ip[1m])", 3.0, 0.2, 0.1), - ("entropy_over_time(flows_src_ip[1m])", entropy, 0.01, 0.01), - ( - "l2_over_time(flows_src_ip[1m])", - 14.0_f64.sqrt(), - 0.01, - 0.01, - ), - ] { - let accuracy = AccuracyTarget::max_error(epsilon).failure_probability(delta); - let root = lower_promql(query, accuracy.clone()).unwrap(); - let root = promql_rows::with_series_identity(&root).unwrap(); - let mut executed = Vec::new(); - // This is an operator execution test. Enumerate each local rule once, - // not an arbitrary sized configuration visited by a budgeted search. - for candidate in explicit_local_candidates(root) { - let family = built_family(&candidate); - let name = match &family { - None => "exact".to_string(), - Some(FieldDataType::Sketch(kind, _)) - if kind.algorithm() == &SketchAlgorithm::UnivMon => - { - "UnivMon".to_string() - } - _ => continue, - }; - if query.starts_with("distinct_over_time") { - if let Some(FieldDataType::Sketch(kind, _)) = &family { - let asap_types::ir::schema::SketchParams::UnivMon { - distinct_failure_probability, - .. - } = kind.params() - else { - unreachable!("family filtered above") - }; - assert_eq!(*distinct_failure_probability, Some(delta)); - } - } - let values = execute(&candidate); - assert!( - matches!(values.as_slice(), [value] if (value - expected).abs() < 1e-9), - "{query} {name}: {values:?}, expected {expected}" - ); - executed.push(name); - } - executed.sort(); - assert_eq!(executed, ["UnivMon", "exact"], "{query}"); - } -} diff --git a/crates/integration-tests/tests/frontend_timestamps.rs b/crates/integration-tests/tests/frontend_timestamps.rs index 5a1fd282b..3fe8f7d61 100644 --- a/crates/integration-tests/tests/frontend_timestamps.rs +++ b/crates/integration-tests/tests/frontend_timestamps.rs @@ -2,7 +2,7 @@ use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::fixtures::lower_promql_root; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, ScalarExpr}; @@ -20,7 +20,7 @@ async fn promql_eval_time_and_sql_current_timestamp_remain_distinct() { "expected a bare evaluation-time scalar, got {promql:?}" ); assert_eq!( - ScalarExpr::EvalTimestamp + ScalarExpr::>::EvalTimestamp .scalar_type(&Schema::default()) .unwrap() .0, @@ -31,13 +31,9 @@ async fn promql_eval_time_and_sql_current_timestamp_remain_distinct() { "metrics", Schema::new(vec![Field::plain("value", DataType::Float64, false)]), ); - let sql = lower_sql( - "SELECT CURRENT_TIMESTAMP FROM metrics", - &catalog, - AccuracyTarget::exact(), - ) - .await - .expect("lower SQL CURRENT_TIMESTAMP"); + let sql = lower_sql("SELECT CURRENT_TIMESTAMP FROM metrics", &catalog) + .await + .expect("lower SQL CURRENT_TIMESTAMP"); let Some(NonASAPOp::Project { cols, child, .. }) = sql.non_asap() else { panic!("expected SQL projection, got {sql:?}"); }; diff --git a/crates/integration-tests/tests/having_shared_aggregate.rs b/crates/integration-tests/tests/having_shared_aggregate.rs deleted file mode 100644 index 554e63d38..000000000 --- a/crates/integration-tests/tests/having_shared_aggregate.rs +++ /dev/null @@ -1,115 +0,0 @@ -//! A key-only `HAVING` of one query is not pushed below an aggregate it -//! shares with another query (#1089): every candidate of the shared -//! aggregate's memo group returns the group's own groups. -use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; -use asap_executor::values::Value; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, -}; -use asap_types::ir::NonASAPOp; -use asap_types::stages::Memo; -use serde_json::json; -use std::path::Path; - -use crate::physical_common; - -const P90: &str = "percentile_cont(0.9) WITHIN GROUP (ORDER BY value)"; -const WINDOW: &str = "ts > now() - INTERVAL '1' HOUR"; - -/// `queries` over `metrics(ts, city, host, value)`, each repeated with -/// `max_error` 0.01 and `failure_probability` 0.001. -fn workload(queries: &[String]) -> FuzzWorkload { - let mut w = read_seed_file( - &Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/sharing_issue_fixtures/902.json"), - ) - .unwrap(); - w.tables = vec![json!({"name":"metrics", "time_index":0, "columns":[ - {"name":"ts","type":"timestamp","nullable":false}, - {"name":"city","type":"utf8","nullable":true}, - {"name":"host","type":"utf8","nullable":true}, - {"name":"value","type":"float64","nullable":true} - ]})]; - let template = w.queries[0].clone(); - w.queries = queries - .iter() - .map(|query| { - let mut entry = template.clone(); - entry.entry.query.0 = query.clone(); - entry - }) - .collect(); - w -} - -/// The Pass 1 memo of `queries` with every Pass 2 sharing rule applied. -fn expanded(queries: &[String]) -> Memo { - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(queries)) - .unwrap() - .into_iter() - .collect(); - let dag = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates(&dag).unwrap(); - let memo = try_populate_pass1(dag, &inventory).unwrap(); - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap() -} - -fn row(city: &str, host: &str, value: f64) -> Vec { - vec![ - Value::Timestamp(0), - Value::Utf8(city.into()), - Value::Utf8(host.into()), - Value::Float64(value), - ] -} - -/// The cities `node` returns over the #1089 rows, sorted. -fn cities(node: &std::rc::Rc) -> Vec { - let rows = vec![ - row("NYC", "h1", 10.0), - row("NYC", "h2", 20.0), - row("LA", "h1", 5.0), - row("LA", "h2", 7.0), - row("SF", "h1", 100.0), - ]; - let mut cities: Vec = physical_common::execute_raw_rows_at(node, rows, 1000) - .into_iter() - .map(|row| format!("{:?}", row[0])) - .collect(); - cities.sort(); - cities -} - -/// #1089: A (`HAVING city = 'NYC'`) listed before B (the same query without -/// it) share one aggregate group. Every candidate of that group returns all -/// three cities, as B needs; A filters above it. -#[test] -fn key_only_having_of_one_reader_stays_above_a_shared_aggregate() { - let grouped = format!("SELECT city, {P90} FROM metrics WHERE {WINDOW} GROUP BY city"); - let memo = expanded(&[format!("{grouped} HAVING city = 'NYC'"), grouped]); - let shared = memo - .groups - .iter() - .position(|group| { - group.reached_by_queries() == [0, 1] - && matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) - }) - .expect("the aggregate both queries read"); - let expected = cities(&memo.groups[shared].logical_node); - assert_eq!(expected.len(), 3, "{expected:?}"); - assert!(memo.candidates[shared].len() > 1, "summary candidates too"); - for candidate in &memo.candidates[shared] { - assert_eq!( - cities(&candidate.root_node), - expected, - "{:?} candidate {:?}", - candidate.origin, - candidate.root_node - ); - } -} diff --git a/crates/integration-tests/tests/identical_exact_queries.rs b/crates/integration-tests/tests/identical_exact_queries.rs index 55fb53010..95b550dc3 100644 --- a/crates/integration-tests/tests/identical_exact_queries.rs +++ b/crates/integration-tests/tests/identical_exact_queries.rs @@ -35,11 +35,11 @@ fn cost(w: &FuzzWorkload) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); - let selection = run.run.selection.as_ref().unwrap(); - let value = selection.cost.cost_per_second.unwrap(); + let (plan, _) = run.run.selection.as_ref().unwrap(); + let value = + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings); println!("cost={value}"); value } diff --git a/crates/integration-tests/tests/independent_plan_seed.rs b/crates/integration-tests/tests/independent_plan_seed.rs index 7dcee170e..4faffc3a5 100644 --- a/crates/integration-tests/tests/independent_plan_seed.rs +++ b/crates/integration-tests/tests/independent_plan_seed.rs @@ -51,12 +51,10 @@ fn cost(queries: &[String]) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); - let selection = run.run.selection.as_ref().unwrap(); - println!("{queries:?}: status {:?}", selection.search_report.status); - selection.cost.cost_per_second.unwrap() + let (plan, _) = run.run.selection.as_ref().unwrap(); + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings) } /// Asserts the joint plan of `queries` costs no more than their separate diff --git a/crates/integration-tests/tests/joint_search_equivalence.rs b/crates/integration-tests/tests/joint_search_equivalence.rs index edd0dd2a3..71b969cce 100644 --- a/crates/integration-tests/tests/joint_search_equivalence.rs +++ b/crates/integration-tests/tests/joint_search_equivalence.rs @@ -7,12 +7,11 @@ use crate::planner_layering_common; -use asap_accuracy::{AccuracyModel, BuiltinAccuracyModel}; use asap_cost::{ClusterLimits, ClusterObjective, CostInputs, CostModel}; use asap_devtools::planning_stages::{lower_promql_roots, lower_sql_roots}; -use asap_logical_asap_optimization::DeploymentCapabilities; use asap_planner::{run_planning, PlanSearchSettings, PlanningModels, PlanningRun}; -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; +use asap_types::accuracy::AccuracyTarget; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::QueryRoot; use asap_types::workload::{ DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, PrometheusMetricType, @@ -44,6 +43,7 @@ fn workload_of(queries: &[(&str, u32)], accuracy: AccuracyTarget) -> PlanningWor data_workload: Some(DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 24.0, + max_out_of_order_ms: None, }, sources: ["m", "n", "c_total"] .into_iter() @@ -86,28 +86,13 @@ fn run( } fn assert_valid(run: &PlanningRun, queries: usize) { - let selection = run + let (plan, selection) = run .selection .as_ref() .unwrap_or_else(|error| panic!("joint planning failed: {error}")); - let physical = &run - .selected_physical_plan - .as_ref() - .expect("successful selection has a physical plan") - .plan; - physical.dag.validate().unwrap(); - assert_eq!(physical.roots.len(), queries); + plan.validate().unwrap(); + assert_eq!(plan.roots.len() + plan.scalar_roots.len(), queries); assert_eq!(selection.accepted_accuracy_per_query.len(), queries); - let report = &selection.search_report; - if let (Some(bound), Some(cost)) = ( - report.cost_lower_bound_per_second, - selection.cost.cost_per_second, - ) { - assert!( - bound <= cost + 1e-8 * cost.abs().max(1e-12), - "reported bound {bound} exceeds validated cost {cost}: {report:?}" - ); - } } fn run_workload( @@ -142,12 +127,10 @@ fn repeated_queries_preserve_all_outputs_and_exact_guarantees() { AccuracyTarget::exact(), ); let run = run_workload(&workload, &settings, &deployment); - let selection = run.selection.as_ref().unwrap(); + let (_, selection) = run.selection.as_ref().unwrap(); let accepted = &selection.accepted_accuracy_per_query; assert_eq!(accepted.len(), count); for accuracy in accepted { - assert!(!accuracy.accepted_regardless_of_guarantee); - assert_eq!(accuracy.guarantee_check_rejection, None); assert_eq!(accuracy.accepted_metric, None, "every output remains exact"); } } @@ -228,35 +211,6 @@ fn composed_quantile_and_sum_outputs_are_validated_together() { ); } -// A custom accuracy policy still participates in final evaluation and prevents -// builtin-only permanent pruning assumptions from being applied to its outputs. -#[test] -fn custom_accuracy_policy_is_respected() { - struct CustomAccuracy(std::cell::Cell); - impl AccuracyModel for CustomAccuracy { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - self.0.set(self.0.get() + 1); - BuiltinAccuracyModel.satisfies(guarantee, target) - } - } - let workload = workload_of(&[("sum_over_time(m[1m])", 60_000)], AccuracyTarget::exact()); - let roots = lower_promql_roots(&workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let policy = CustomAccuracy(std::cell::Cell::new(0)); - let result = run( - &roots, - &entries, - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin().with_accuracy(&policy), - false, - ); - assert_valid(&result, 1); - assert!( - policy.0.get() > 0, - "the supplied accuracy policy must evaluate outputs" - ); -} - // The SQL frontend keeps repeated and different-window COUNT DISTINCT outputs. #[test] fn repeated_sql_queries_preserve_separate_output_references() { @@ -308,7 +262,6 @@ fn rootless_request_overhead_respects_capacity_and_latency() { false, ); assert!(result.selection.is_err()); - assert!(result.selected_physical_plan.is_none()); } } @@ -330,7 +283,6 @@ fn unsupported_exact_operation_never_returns_an_unvalidated_plan() { false, ); assert!(result.selection.is_err()); - assert!(result.selected_physical_plan.is_none()); } // Deterministic small mixed workloads exercise exact temporal operators, counter diff --git a/crates/integration-tests/tests/kll_tumbling_window_execution.rs b/crates/integration-tests/tests/kll_tumbling_window_execution.rs index bc0ad0698..48e72c8c1 100644 --- a/crates/integration-tests/tests/kll_tumbling_window_execution.rs +++ b/crates/integration-tests/tests/kll_tumbling_window_execution.rs @@ -10,21 +10,14 @@ use asap_executor::{ values::{Batch, SchemaRef, Value}, AggregateCore, Error, }; -use asap_types::ir::materialized_view::MaterializedView; -use asap_types::ir::operator::operator_properties::TimeShift; +use asap_types::ir::operator::arguments::TimeShift; use asap_types::ir::operator::{GroupKeys, Reduction, Source}; -use asap_types::ir::physical_export::{ - compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG, -}; use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, SummaryUpdate, -}; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, NonASAPOp, - OperatorNode, QueryRoot, TimeRangeKind, -}; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; +use asap_types::plan::materialized_view::MaterializedView; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; use futures::{executor::block_on, StreamExt}; use std::{ rc::Rc, @@ -36,17 +29,8 @@ use std::{ const TUMBLING_WINDOW_WIDTH_MS: i64 = 60_000; -fn family(k: u32) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - Default::default(), - ) +fn family(_k: u32) -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll) } /// `t(ts, value)`, time index `ts`. fn table_schema() -> Schema { @@ -101,17 +85,15 @@ fn fixture() -> Fixture { })); new(asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { child: range, - family: family(200), + summary_type: family(200), input: SummaryUpdate::column(ColumnRef::Named("value".into())), reduction: Reduction::Reduce(GroupKeys::default()), - grouping: GroupingStrategy::PerSubpopulationInstance, filter: None, })) }) .collect(); let merge = new(asap_types::ir::Operator::ASAP(ASAPOp::SummaryMerge { children: tumbling_windows.clone(), - newest_tumbling_windows: None, group_by: None, })); let estimate = |q| { @@ -120,20 +102,17 @@ fn fixture() -> Fixture { query: SketchStatistic::Quantile { q }, })) }; - let roots = apply_materialization_timings_to_roots( - &[ - QueryRoot::Operator(estimate(0.5)), - QueryRoot::Operator(estimate(0.99)), - ], - &MaterializationAssignment::default(), - ) - .unwrap(); - let compiled = compile_physical_asap_workload_with_node_ids(&roots).unwrap(); + let roots = [ + QueryRoot::Operator(estimate(0.5)), + QueryRoot::Operator(estimate(0.99)), + ]; + let (dag, operator_node_of_id) = + asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots).unwrap(); let mut tumbling_window_ids = vec![0; 5]; let (mut merge, mut p50, mut p99) = (0, 0, 0); - for node in &compiled.dag.nodes { + for node in &dag.nodes { let id = node.id as u64; - match &compiled.node_ids.operator_node(node.id).unwrap().operator { + match &operator_node_of_id[node.id].operator { asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => { let shift = match child.children()[0].non_asap() { Some(NonASAPOp::TimeShift { shift, .. }) => shift.offset_ms, @@ -152,7 +131,7 @@ fn fixture() -> Fixture { _ => {} } } - let mut dag = compiled.dag; + let mut dag = dag; let views = views_at(&mut dag, &tumbling_window_ids).unwrap(); Fixture { dag, diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index bf81ee35d..fa844281e 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -8,52 +8,26 @@ mod planner_layering_common; mod aggregate; mod binary_op; mod crate_graph; -mod declared_group_counts; -mod deployment_capabilities; -mod exact_composition; mod filtered_aggregates; mod finer_aggregate_reuse; -mod frequency_execution; mod frontend_timestamps; -mod having_shared_aggregate; mod identical_exact_queries; mod independent_plan_seed; mod joint_search_equivalence; mod kll_tumbling_window_execution; -mod materialization_e2e; mod materialized_views; -mod memo_enumeration; mod nested; mod operator_design_examples; mod operator_sharing; mod per_computation_candidates_sql; mod per_series_exact_execution; -mod planner_layering_example1; -mod planner_layering_example2; -mod planner_layering_example3; -mod planner_layering_example4; -mod precompute_raw_samples; mod production_rollup_workloads; -mod promql_numeric_regressions; mod promql_to_post_asap; -mod range_evaluation; -mod reuse_rules; mod rollup_controlled; -mod runtime_handoff; mod scan; mod schema; mod sharing_issue_regressions; -mod sketch_rollup; -mod sketch_rollup_planning; mod sql_cardinality; mod sql_frequency; -mod sql_group_bound; -mod sql_to_physical; mod sql_to_post_asap; -mod sql_trailing_window; -mod sql_two_percentiles; -mod stage1_logical_asap_optimization; -mod stage2_physical_asap_optimization_sql; -mod summary_coverage_e2e; mod time_range; -mod tumbling_window_merge; diff --git a/crates/integration-tests/tests/materialization_e2e.rs b/crates/integration-tests/tests/materialization_e2e.rs deleted file mode 100644 index 1751d3e95..000000000 --- a/crates/integration-tests/tests/materialization_e2e.rs +++ /dev/null @@ -1,356 +0,0 @@ -//! Materialization decides placement, not answers: the same selected DAG -//! timed with its states `NotMaterialized` (everything at query time) and -//! `MatAtIngestion` (materialized views refreshed at ingestion time) -//! executes to the same result. - -use std::collections::{BTreeMap, HashMap}; -use std::rc::Rc; - -use asap_executor::physical_planner::{compile, promql_rows}; -use asap_executor::runtime::Scope; -use asap_executor::values::{Batch, SchemaRef, Value}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{ - explicit_local_candidates, explicit_summary_candidate, selected_dag, -}; -use asap_physical_asap_optimization::test_support::split_shared_by_timing; -use asap_types::ir::operator::Source; -use asap_types::ir::physical_export::{ - compile_physical_asap_dag, PhysicalASAPDAG, PhysicalASAPOperatorPayload, -}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, OperatorNode, -}; -use asap_types::physical::{Materialization, Medium}; - -use crate::physical_common; - -const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - -/// The logical DAG the planner pipeline selects for `query`, optionally with -/// a PromQL series identity (per-entity state needs one). -fn selected(query: &str, series_identity: bool) -> Rc { - selected_with(query, series_identity, AccuracyTarget::max_error(0.01)) -} - -/// [`selected`] for a quantile query, whose target names its metric (a -/// rank error). -fn selected_quantile(query: &str, series_identity: bool) -> Rc { - selected_with( - query, - series_identity, - AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank), - ) -} - -fn selected_with(query: &str, series_identity: bool, accuracy: AccuracyTarget) -> Rc { - let mut root = lower_promql(query, accuracy.clone()).unwrap(); - if series_identity { - root = promql_rows::with_series_identity(&root).unwrap(); - } - selected_dag(root, accuracy) -} - -/// The first logical candidate for `query` that builds a sketch: plan -/// selection picks the exact plan while every summary runs at query time. -fn sketch_candidate(query: &str) -> Rc { - let accuracy = AccuracyTarget::max_error(0.01); - let root = lower_promql(query, accuracy.clone()).unwrap(); - explicit_local_candidates(root) - .into_iter() - .find(|candidate| { - OperatorNode::reachable(candidate).iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(..), - .. - }) - ) - }) - }) - .expect("a sketch candidate") -} - -/// The two placements under test: nothing stored, and every state stored at -/// ingestion time where the DAG allows it. -fn placements() -> [MaterializationAssignment; 2] { - [ - MaterializationAssignment::default(), - MaterializationAssignment::with_default(AT_INGESTION), - ] -} - -/// `root` timed under `assignment`, as Stage 2 times a plan: a node read at -/// two timings (Pass 1 composes identical inputs as one memo group's node, -/// e.g. the scan of two quantiles of one metric) is split first, one copy -/// per timing. -fn timed_dag(root: &Rc, assignment: &MaterializationAssignment) -> PhysicalASAPDAG { - let (split, assignment) = split_shared_by_timing(std::slice::from_ref(root), assignment); - let timed = - apply_materialization_timings(&split[0], &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() -} - -fn states(dag: &PhysicalASAPDAG) -> Vec { - dag.nodes - .iter() - .filter(|node| node.materialization == AT_INGESTION) - .map(|node| node.id as u64) - .collect() -} - -fn floats(result: &[Vec]) -> Vec { - result[0] - .iter() - .flat_map(|batch| batch.to_rows()) - .flatten() - .filter_map(|value| match value { - Value::Float64(value) => Some(value), - _ => None, - }) - .collect() -} - -/// Execute a timed DAG over one raw-source batch per `Scan` -/// (`batch(schema, metric)`): each refresh group of materialized views -/// refreshed in one ingestion run over `ingestion`, then the query at `evaluation_time_ms` -/// reading the views' states. Returns the root's values and the views. -fn execute( - dag: &PhysicalASAPDAG, - batch: &dyn Fn(&SchemaRef, &str) -> Batch, - ingestion: Scope, - evaluation_time_ms: i64, -) -> (Vec, Vec) { - let mut dag = dag.clone(); - let views = dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let raw = |dag: &PhysicalASAPDAG| { - physical_common::raw_data_store(dag, |source, schema| { - let Source::TimeSeries { metric } = source else { - panic!("a time series source") - }; - batch(schema, metric).to_rows() - }) - }; - // Each refresh group's ingestion run refreshes its views from the raw - // data, compiled once with all of them as roots. - let mut store = raw(&dag); - for group in dag.ingestion_refresh_groups().unwrap() { - let roots: Vec = group.views.iter().map(|view| view.node_id as u64).collect(); - let refresh = compile(&dag, &[], &roots).unwrap(); - let states = physical_common::execute(&refresh, raw(&dag), ingestion.clone()); - for (view, states) in group.views.iter().zip(states) { - store = store.with_materialized_view(view.node_id, states).unwrap(); - } - } - let query = compile(&dag, &views, &[dag.roots[0] as u64]).unwrap(); - let result = physical_common::execute(&query, store, Scope::Query { evaluation_time_ms }); - ( - floats(&result), - views.iter().map(|view| view.node_id as u64).collect(), - ) -} - -/// `(metric, job, seconds, value)` samples as one batch per metric. -fn job_samples<'a>( - samples: &'a [(&'a str, &'a str, i64, f64)], -) -> impl Fn(&SchemaRef, &str) -> Batch + 'a { - move |schema, name| { - let rows = samples - .iter() - .filter(|sample| sample.0 == name) - .map(|(metric, job, seconds, value)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), metric.to_string()), - ("job".to_string(), job.to_string()), - ]); - promql_rows::series_row(schema, &labels, seconds * 1000, *value).unwrap() - }) - .collect(); - Batch::try_new(schema.clone(), rows).unwrap() - } -} - -/// Values 1..=100 at 300 s, one row each, in every Float64 column of -/// `schema`. -fn values_1_to_100(schema: &SchemaRef, _metric: &str) -> Batch { - let rows = (1..=100) - .map(|value| { - schema - .fields - .iter() - .map(|field| match field.dtype { - asap_types::ir::schema::FieldDataType::Plain( - asap_types::ir::schema::DataType::Float64, - ) => Value::Float64(f64::from(value)), - asap_types::ir::schema::FieldDataType::Plain( - asap_types::ir::schema::DataType::Timestamp, - ) => Value::Timestamp(300_000), - _ => panic!("unexpected field {field:?}"), - }) - .collect() - }) - .collect(); - Batch::try_new(schema.clone(), rows).unwrap() -} - -/// `MatAtIngestion` puts the quantile's state in a materialized view; -/// `NotMaterialized` has no view and reads the raw source at -/// query time; both answer alike. -#[test] -fn materialization_decides_precompute_contents() { - let root = sketch_candidate("quantile(0.99, latency)"); - let mut answers = Vec::new(); - for assignment in placements() { - let dag = timed_dag(&root, &assignment); - let ingestion = Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 300_000, - }; - let (values, frontier) = execute(&dag, &values_1_to_100, ingestion, 300_000); - assert_eq!(frontier, states(&dag)); - answers.push(values); - } - assert_eq!(answers[1].len(), 1); - assert!(!states(&timed_dag(&root, &placements()[1])).is_empty()); - assert_eq!(answers[0], answers[1]); -} - -/// Mixed placement in one DAG: one summary `MatAtIngestion` (a materialized view) -/// and the other `NotMaterialized` (built from the raw source at query -/// time) answer like the all-query-time placement. -#[test] -fn mixed_placement_answers_like_query_time() { - let accuracy = AccuracyTarget::max_error(0.01); - let root = lower_promql( - "quantile(0.99, latency) + quantile(0.5, latency)", - accuracy.clone(), - ) - .unwrap(); - let sketches = |candidate: &Rc| -> Vec> { - OperatorNode::reachable(candidate) - .into_iter() - .filter(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(..), - .. - }) - ) - }) - .collect() - }; - let root = explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::Kll); - assert_eq!( - sketches(&root).len(), - 2, - "two explicitly selected KLL builds" - ); - let mut mixed = MaterializationAssignment::default(); - mixed.set(&sketches(&root)[0], AT_INGESTION); - let mut answers = Vec::new(); - for assignment in [MaterializationAssignment::default(), mixed] { - let dag = timed_dag(&root, &assignment); - let ingestion = Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 300_000, - }; - let (values, frontier) = execute(&dag, &values_1_to_100, ingestion, 300_000); - assert_eq!(frontier, states(&dag)); - answers.push((values, frontier.len())); - } - // Only the assigned sketch is stored. - assert_eq!(answers[0].1, 0); - assert_eq!(answers[1].1, 1); - assert_eq!(answers[1].0.len(), 1); - assert_eq!(answers[0].0, answers[1].0); -} - -/// Whether the DAG has an ingestion-time `BinaryOp`. -fn ingestion_binary(dag: &PhysicalASAPDAG) -> bool { - dag.nodes.iter().any(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::BinaryOp { .. }) - ) && node.output_state.timing == ExecutionTiming::IngestionTime - }) -} - -const INGESTION_WINDOW: Scope = Scope::Ingestion { - window_start_ms: 240_000, - window_end_ms: 300_000, -}; - -/// Prometheus drops series without a match: arithmetic over different -/// selectors keeps only label sets present on both sides (none when -/// disjoint), at either placement, and never becomes aligned maintenance. -#[test] -fn arithmetic_over_different_selectors_matches_prometheus() { - // The selected root reads the outer `sum`'s exact state through its - // finalize. - let root = selected("sum(sum_over_time(m[1m]) + sum_over_time(n[1m]))", true); - assert!(matches!( - root.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - )); - let disjoint = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("n", "b", 250, 5.0), - ]; - // Only job a is on both sides: m_a + n_a = (1 + 2) + 7; m{job="b"} is dropped. - let overlapping = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("m", "b", 250, 5.0), - ("n", "a", 250, 7.0), - ]; - for assignment in placements() { - let dag = timed_dag(&root, &assignment); - assert!(!ingestion_binary(&dag)); - let (values, _) = execute(&dag, &job_samples(&disjoint), INGESTION_WINDOW, 300_000); - assert!(values.is_empty(), "{values:?}"); - let (values, _) = execute(&dag, &job_samples(&overlapping), INGESTION_WINDOW, 300_000); - assert_eq!(values, [10.0]); - } - let root = selected_quantile( - "quantile(0.9, sum_over_time(m[1m]) + sum_over_time(n[1m]))", - true, - ); - assert!(!ingestion_binary(&timed_dag(&root, &placements()[1]))); -} - -/// Arithmetic over one selector is aligned maintenance when stored at -/// ingestion time, and adds each series' two evaluations before the -/// quantile at either placement. -#[test] -#[ignore = "the two identical selectors stay separate until Pass 2 merges identical sub-DAGs (#580)"] -fn arithmetic_over_one_selector_executes() { - let root = selected_quantile( - "quantile(0.9, sum_over_time(m[1m]) + sum_over_time(m[1m]))", - true, - ); - let samples = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("m", "b", 250, 5.0), - ]; - let [not_materialized, at_ingestion] = placements().map(|a| timed_dag(&root, &a)); - assert!(!ingestion_binary(¬_materialized)); - assert!( - ingestion_binary(&at_ingestion), - "one selector shares its key set" - ); - for dag in [not_materialized, at_ingestion] { - let (values, _) = execute(&dag, &job_samples(&samples), INGESTION_WINDOW, 300_000); - // job a: 3 + 3 = 6; job b: 5 + 5 = 10 (mispairing a with b gives 8 and 8). - // KLL at epsilon 0.01 returns an input value within 0.01 of rank 0.9; - // of two values only the larger is. - assert_eq!(values, [10.0]); - } -} diff --git a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json index 515db4302..53ef51dfd 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json @@ -1 +1 @@ -{"dag":{"schema_version":10,"dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]}},"queries":[{"query":"sum by (job) (sum_over_time(m[1m]))","root_node_id":5},{"query":"sum by (job, instance) (sum_over_time(m[1m]))","root_node_id":6}],"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}],"query_time_raw_windows":[],"selection":null} +{"dag": {"schema_version": 10, "dag": {"edges": [{"consumer": 1, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 0, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 2, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 1, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 3, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 4, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 3, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 5, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "producer": 4, "role": "Input", "window": "NotApplicable"}, {"consumer": 6, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "NotApplicable"}], "nodes": [{"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 0, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"Scan": {"predicates": [], "schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "source": {"TimeSeries": {"metric": "m"}}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 1, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"TimeRange": {"child": 0, "kind": "range", "range": {"nanos": 0, "secs": 60}}}}}, {"guarantee": null, "id": 2, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 1, "filter": null, "input": {"item": null, "weight": {"Column": "SampleValue"}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [2, 3]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 3, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}, {"guarantee": null, "id": 4, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 3, "filter": null, "input": {"item": null, "weight": {"Column": {"Named": "sum"}}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [1]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 5, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 4}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 6, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}], "roots": [5, 6], "scalar_roots": []}}, "queries": [{"query": "sum by (job) (sum_over_time(m[1m]))", "root_node_id": 5}, {"query": "sum by (job, instance) (sum_over_time(m[1m]))", "root_node_id": 6}], "stored_outputs": [{"built_at": "ingestion_run", "coverage": {"definition_id": "db0c9241a79202b6925e84aef010e89a", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [{"end_offset_ms": 0, "scan_node_id": 0, "time_range_node_id": 1, "width_ms": 60000}], "stored_outputs": []}, "node_id": 2, "retention_ms": 60000}, {"built_at": "ingestion_run", "coverage": {"definition_id": "3c90a6b5ff3ecaa445ea9bf285e1dc0b", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [], "stored_outputs": [2]}, "node_id": 4, "retention_ms": 60000}], "query_time_raw_windows": [], "selection": null} diff --git a/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json b/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json index 3a73c4fcf..f822d6b70 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json @@ -1,10 +1,10 @@ [ -{"name":"repeating p99-60m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":35,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":37,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":36,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":38,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":37,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":39,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":40,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":39,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":41,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":40,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":42,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":43,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":42,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":44,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":43,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":45,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":46,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":45,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":47,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":46,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":48,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":49,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":48,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":50,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":49,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":51,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":52,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":51,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":53,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":52,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":38,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":41,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":44,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":47,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":50,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":53,"role":"Input","window":"NotApplicable"},{"consumer":55,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":54,"role":"Input","window":"NotApplicable"},{"consumer":56,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":55,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":200000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":400000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":800000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1000000}}}}},{"guarantee":null,"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":17,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1200000}}}}},{"guarantee":null,"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":20,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1400000}}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":23,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1600000}}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":26,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1800000}}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":29,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2000000}}}}},{"guarantee":null,"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":30,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":32,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":33,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2200000}}}}},{"guarantee":null,"id":34,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":33,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":35,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":34,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":36,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2400000}}}}},{"guarantee":null,"id":37,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":36,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":38,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":37,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":39,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2600000}}}}},{"guarantee":null,"id":40,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":39,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":41,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":40,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":42,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2800000}}}}},{"guarantee":null,"id":43,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":42,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":44,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":43,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":45,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3000000}}}}},{"guarantee":null,"id":46,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":45,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":47,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":46,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":48,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3200000}}}}},{"guarantee":null,"id":49,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":48,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":50,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":49,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":51,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3400000}}}}},{"guarantee":null,"id":52,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":51,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":53,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":52,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":54,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47,50,53]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":55,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":54}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":56,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":55,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[56],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-200000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":200000}],"stored_outputs":[]},"node_id":2,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-400000},{"included":-200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":200000,"scan_node_id":0,"time_range_node_id":4,"width_ms":200000}],"stored_outputs":[]},"node_id":5,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":400000,"scan_node_id":0,"time_range_node_id":7,"width_ms":200000}],"stored_outputs":[]},"node_id":8,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-800000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":10,"width_ms":200000}],"stored_outputs":[]},"node_id":11,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1000000},{"included":-800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":800000,"scan_node_id":0,"time_range_node_id":13,"width_ms":200000}],"stored_outputs":[]},"node_id":14,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1200000},{"included":-1000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1000000,"scan_node_id":0,"time_range_node_id":16,"width_ms":200000}],"stored_outputs":[]},"node_id":17,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1400000},{"included":-1200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1200000,"scan_node_id":0,"time_range_node_id":19,"width_ms":200000}],"stored_outputs":[]},"node_id":20,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1600000},{"included":-1400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1400000,"scan_node_id":0,"time_range_node_id":22,"width_ms":200000}],"stored_outputs":[]},"node_id":23,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1800000},{"included":-1600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1600000,"scan_node_id":0,"time_range_node_id":25,"width_ms":200000}],"stored_outputs":[]},"node_id":26,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2000000},{"included":-1800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1800000,"scan_node_id":0,"time_range_node_id":28,"width_ms":200000}],"stored_outputs":[]},"node_id":29,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2200000},{"included":-2000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2000000,"scan_node_id":0,"time_range_node_id":31,"width_ms":200000}],"stored_outputs":[]},"node_id":32,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2400000},{"included":-2200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2200000,"scan_node_id":0,"time_range_node_id":34,"width_ms":200000}],"stored_outputs":[]},"node_id":35,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2600000},{"included":-2400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2400000,"scan_node_id":0,"time_range_node_id":37,"width_ms":200000}],"stored_outputs":[]},"node_id":38,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2800000},{"included":-2600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2600000,"scan_node_id":0,"time_range_node_id":40,"width_ms":200000}],"stored_outputs":[]},"node_id":41,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3000000},{"included":-2800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2800000,"scan_node_id":0,"time_range_node_id":43,"width_ms":200000}],"stored_outputs":[]},"node_id":44,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3200000},{"included":-3000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3000000,"scan_node_id":0,"time_range_node_id":46,"width_ms":200000}],"stored_outputs":[]},"node_id":47,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3400000},{"included":-3200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3200000,"scan_node_id":0,"time_range_node_id":49,"width_ms":200000}],"stored_outputs":[]},"node_id":50,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3600000},{"included":-3400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3400000,"scan_node_id":0,"time_range_node_id":52,"width_ms":200000}],"stored_outputs":[]},"node_id":53,"retention_ms":3600000}]}, -{"name":"repeating offset-chain/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":2,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":1,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":3,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":2,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":4,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":6,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":5,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":8,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":7,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":9,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":11,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":10,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":12,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":11,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":14,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":13,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":15,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":14,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":17,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":16,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":18,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":17,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":660000}}}}},{"guarantee":null,"id":20,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":720000}}}}},{"guarantee":null,"id":23,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":780000}}}}},{"guarantee":null,"id":26,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":840000}}}}},{"guarantee":null,"id":29,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[3,6,9,12,15,18,21,24,27,30]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":32,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":31}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":33,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":32,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[33],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":2,"width_ms":60000}],"stored_outputs":[]},"node_id":3,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":5,"width_ms":60000}],"stored_outputs":[]},"node_id":6,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":8,"width_ms":60000}],"stored_outputs":[]},"node_id":9,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":11,"width_ms":60000}],"stored_outputs":[]},"node_id":12,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":14,"width_ms":60000}],"stored_outputs":[]},"node_id":15,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-660000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":17,"width_ms":60000}],"stored_outputs":[]},"node_id":18,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-720000},{"included":-660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":660000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-780000},{"included":-720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":720000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-840000},{"included":-780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":780000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-900000},{"included":-840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":840000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":900000}]}, -{"name":"repeating rollup/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":34,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":17,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":20,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":23,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":26,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":29,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":30}}}},{"guarantee":null,"id":32,"materialization":{"MatAtQuery":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":33,"materialization":{"MatAtQuery":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":32}}}},{"guarantee":null,"id":34,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":33,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}}],"roots":[33,35],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":16,"width_ms":60000}],"stored_outputs":[]},"node_id":17,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":19,"width_ms":60000}],"stored_outputs":[]},"node_id":20,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":22,"width_ms":60000}],"stored_outputs":[]},"node_id":23,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":25,"width_ms":60000}],"stored_outputs":[]},"node_id":26,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":28,"width_ms":60000}],"stored_outputs":[]},"node_id":29,"retention_ms":600000}]}, -{"name":"repeating rate-5m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"NotApplicable"},{"consumer":18,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"NotApplicable"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":37,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":36,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}},{"guarantee":null,"id":17,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":18,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":17}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3600000}}}}},{"guarantee":null,"id":20,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3660000}}}}},{"guarantee":null,"id":23,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3720000}}}}},{"guarantee":null,"id":26,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3780000}}}}},{"guarantee":null,"id":29,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3840000}}}}},{"guarantee":null,"id":32,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":31,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":33,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":32,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":34,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[21,24,27,30,33]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}},{"guarantee":null,"id":36,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":35,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":37,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":36}}}}],"roots":[18,37],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3660000},{"included":-3600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3600000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3720000},{"included":-3660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3660000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3780000},{"included":-3720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3720000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3840000},{"included":-3780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3780000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3900000},{"included":-3840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3840000,"scan_node_id":0,"time_range_node_id":32,"width_ms":60000}],"stored_outputs":[]},"node_id":33,"retention_ms":3900000}]}, -{"name":"runtime_install a_coarse_sum_rolls_up_from_the_stored_fine_sum","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, -{"name":"runtime_install a_kept_sliding_window_merge_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"NotApplicable"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"NotApplicable"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"NotApplicable"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}}],"roots":[16],"scalar_roots":[]},"stored_outputs":[{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2,5,8,11,14]},"node_id":15,"retention_ms":3900000}]}, -{"name":"runtime_install a_summary_across_series_from_per_series_summaries_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":6,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":3}}],"qualifier":null}}}}],"roots":[5,7],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"5555de6812a18c62345c199d5ba8f06e","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"f35ee4e847802c130cbd5221a81f7ea1","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, -{"name":"runtime_install group_outputs_from_one_stored_per_series_sum_are_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":9,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":3,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":5}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":8,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":7,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"instance","expr":{"Column":3}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":9,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":9}}}}],"roots":[6,10],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"672f965318f2bf2ba350748044719b9b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"7976839b25e74a9cfac2ab1d6d01a63a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":5,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"84209f44520311bced823b9e98eaf85c","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":9,"retention_ms":60000}]} +{"name":"repeating p99-60m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":35,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":37,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":36,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":38,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":37,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":39,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":40,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":39,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":41,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":40,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":42,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":43,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":42,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":44,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":43,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":45,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":46,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":45,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":47,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":46,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":48,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":49,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":48,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":50,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":49,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":51,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":52,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":51,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":53,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":52,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":38,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":41,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":44,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":47,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":50,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":53,"role":"Input","window":"NotApplicable"},{"consumer":55,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":54,"role":"Input","window":"NotApplicable"},{"consumer":56,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":55,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":200000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":400000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":800000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1000000}}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1200000}}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1400000}}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1600000}}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1800000}}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2000000}}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":30,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":32,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2200000}}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":33,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":35,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":34,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2400000}}}}},{"guarantee":null,"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":36,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":38,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":37,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":39,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2600000}}}}},{"guarantee":null,"id":40,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":39,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":41,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":40,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":42,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2800000}}}}},{"guarantee":null,"id":43,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":42,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":44,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":43,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":45,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3000000}}}}},{"guarantee":null,"id":46,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":45,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":47,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":46,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":48,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3200000}}}}},{"guarantee":null,"id":49,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":48,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":50,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":49,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":51,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3400000}}}}},{"guarantee":null,"id":52,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":51,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":53,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":52,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":54,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47,50,53]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":55,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":54}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":56,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":55,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[56],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-200000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":200000}],"stored_outputs":[]},"node_id":2,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-400000},{"included":-200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":200000,"scan_node_id":0,"time_range_node_id":4,"width_ms":200000}],"stored_outputs":[]},"node_id":5,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":400000,"scan_node_id":0,"time_range_node_id":7,"width_ms":200000}],"stored_outputs":[]},"node_id":8,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-800000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":10,"width_ms":200000}],"stored_outputs":[]},"node_id":11,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1000000},{"included":-800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":800000,"scan_node_id":0,"time_range_node_id":13,"width_ms":200000}],"stored_outputs":[]},"node_id":14,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1200000},{"included":-1000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1000000,"scan_node_id":0,"time_range_node_id":16,"width_ms":200000}],"stored_outputs":[]},"node_id":17,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1400000},{"included":-1200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1200000,"scan_node_id":0,"time_range_node_id":19,"width_ms":200000}],"stored_outputs":[]},"node_id":20,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1600000},{"included":-1400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1400000,"scan_node_id":0,"time_range_node_id":22,"width_ms":200000}],"stored_outputs":[]},"node_id":23,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1800000},{"included":-1600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1600000,"scan_node_id":0,"time_range_node_id":25,"width_ms":200000}],"stored_outputs":[]},"node_id":26,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2000000},{"included":-1800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1800000,"scan_node_id":0,"time_range_node_id":28,"width_ms":200000}],"stored_outputs":[]},"node_id":29,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2200000},{"included":-2000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2000000,"scan_node_id":0,"time_range_node_id":31,"width_ms":200000}],"stored_outputs":[]},"node_id":32,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2400000},{"included":-2200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2200000,"scan_node_id":0,"time_range_node_id":34,"width_ms":200000}],"stored_outputs":[]},"node_id":35,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2600000},{"included":-2400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2400000,"scan_node_id":0,"time_range_node_id":37,"width_ms":200000}],"stored_outputs":[]},"node_id":38,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2800000},{"included":-2600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2600000,"scan_node_id":0,"time_range_node_id":40,"width_ms":200000}],"stored_outputs":[]},"node_id":41,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3000000},{"included":-2800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2800000,"scan_node_id":0,"time_range_node_id":43,"width_ms":200000}],"stored_outputs":[]},"node_id":44,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3200000},{"included":-3000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3000000,"scan_node_id":0,"time_range_node_id":46,"width_ms":200000}],"stored_outputs":[]},"node_id":47,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3400000},{"included":-3200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3200000,"scan_node_id":0,"time_range_node_id":49,"width_ms":200000}],"stored_outputs":[]},"node_id":50,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3600000},{"included":-3400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3400000,"scan_node_id":0,"time_range_node_id":52,"width_ms":200000}],"stored_outputs":[]},"node_id":53,"retention_ms":3600000}]}, +{"name":"repeating offset-chain/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":2,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":1,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":3,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":2,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":4,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":6,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":5,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":7,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":11,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":10,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":12,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":11,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":14,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":13,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":14,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":16,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":18,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":17,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":660000}}}}},{"guarantee":null,"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":720000}}}}},{"guarantee":null,"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":780000}}}}},{"guarantee":null,"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":840000}}}}},{"guarantee":null,"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[3,6,9,12,15,18,21,24,27,30]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":31}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":32,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[33],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":2,"width_ms":60000}],"stored_outputs":[]},"node_id":3,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":5,"width_ms":60000}],"stored_outputs":[]},"node_id":6,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":8,"width_ms":60000}],"stored_outputs":[]},"node_id":9,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":11,"width_ms":60000}],"stored_outputs":[]},"node_id":12,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":14,"width_ms":60000}],"stored_outputs":[]},"node_id":15,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-660000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":17,"width_ms":60000}],"stored_outputs":[]},"node_id":18,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-720000},{"included":-660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":660000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-780000},{"included":-720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":720000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-840000},{"included":-780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":780000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-900000},{"included":-840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":840000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":900000}]}, +{"name":"repeating rollup/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":34,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":30}}}},{"guarantee":null,"id":32,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":33,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":32}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":33,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}}],"roots":[33,35],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":16,"width_ms":60000}],"stored_outputs":[]},"node_id":17,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":19,"width_ms":60000}],"stored_outputs":[]},"node_id":20,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":22,"width_ms":60000}],"stored_outputs":[]},"node_id":23,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":25,"width_ms":60000}],"stored_outputs":[]},"node_id":26,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":28,"width_ms":60000}],"stored_outputs":[]},"node_id":29,"retention_ms":600000}]}, +{"name":"repeating rate-5m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"NotApplicable"},{"consumer":18,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"NotApplicable"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":37,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":36,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}},{"guarantee":null,"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":17}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3600000}}}}},{"guarantee":null,"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3660000}}}}},{"guarantee":null,"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3720000}}}}},{"guarantee":null,"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3780000}}}}},{"guarantee":null,"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3840000}}}}},{"guarantee":null,"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":31,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":33,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":32,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[21,24,27,30,33]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}},{"guarantee":null,"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":35,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":36}}}}],"roots":[18,37],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3660000},{"included":-3600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3600000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3720000},{"included":-3660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3660000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3780000},{"included":-3720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3720000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3840000},{"included":-3780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3780000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3900000},{"included":-3840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3840000,"scan_node_id":0,"time_range_node_id":32,"width_ms":60000}],"stored_outputs":[]},"node_id":33,"retention_ms":3900000}]}, +{"name":"runtime_install a_coarse_sum_rolls_up_from_the_stored_fine_sum","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, +{"name":"runtime_install a_materialized_sliding_window_merge_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"NotApplicable"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"NotApplicable"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"NotApplicable"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}}],"roots":[16],"scalar_roots":[]},"stored_outputs":[{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2,5,8,11,14]},"node_id":15,"retention_ms":3900000}]}, +{"name":"runtime_install a_summary_across_series_from_per_series_summaries_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":6,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":3}}],"qualifier":null}}}}],"roots":[5,7],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"5555de6812a18c62345c199d5ba8f06e","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"f35ee4e847802c130cbd5221a81f7ea1","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, +{"name":"runtime_install group_outputs_from_one_stored_per_series_sum_are_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":9,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":3,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":5}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":7,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"instance","expr":{"Column":3}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":9}}}}],"roots":[6,10],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"672f965318f2bf2ba350748044719b9b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"7976839b25e74a9cfac2ab1d6d01a63a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":5,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"84209f44520311bced823b9e98eaf85c","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":9,"retention_ms":60000}]} ] diff --git a/crates/integration-tests/tests/materialized_views.rs b/crates/integration-tests/tests/materialized_views.rs index c6fad2f29..1633e5691 100644 --- a/crates/integration-tests/tests/materialized_views.rs +++ b/crates/integration-tests/tests/materialized_views.rs @@ -3,153 +3,21 @@ use std::collections::HashSet; -use asap_logical_asap_optimization::DeploymentCapabilities; use asap_planner::{ - e2e_plan, installable_plan_of, plan_installable, FrontendInput, InstallablePlan, - InstallablePlanError, MaterializedViewRefresh, PlanningModels, RawDataRetention, UserInput, - INSTALLABLE_PLAN_WIRE_VERSION, + InstallablePlan, InstallablePlanError, MaterializedViewRefresh, INSTALLABLE_PLAN_WIRE_VERSION, }; use asap_types::ir::operator::Source; -use asap_types::ir::physical_export::{compile_physical_asap_workload, PhysicalASAPDAG}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; -use asap_types::physical::Materialization; -use asap_types::workload::PlanningWorkload; -use serde_json::Value; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; -/// A repeating PromQL workload over ingesting `m`, each query every minute. -fn repeating(queries: &[&str]) -> PlanningWorkload { - repeating_with(queries, "") -} +use asap_types::ir::{NonASAPOp, Operator}; -/// [`repeating`], each entry with `extra` fields (e.g. a range evaluation). -fn repeating_with(queries: &[&str], extra: &str) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|q| { - format!( - r#"{{"query": {q:?}, "demand": {{"fixed_interval": 60000}}, - "requirements": {{"accuracy": {{"max_error": 0.01, "failure_probability": 0.01}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}{extra}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, - "sources": {{"m": {{"series_count": 1000, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// Stage 3's priced raw lookback of `dag` (the selected candidate's -/// physical DAG, tumbling-window chains unexpanded), per source: each query-time -/// scan's `PhysicalASAPDAG::scan_window` with every chain kept across -/// evaluations reading its newest tumbling window, as `asap-plan-selection` prices -/// it, the longest per source. -fn stage3_raw_lookback_ms(dag: &PhysicalASAPDAG) -> Vec<(Source, Option)> { - let kept_chains: HashSet<_> = dag - .nodes - .iter() - .filter(|node| { - matches!( - (&node.payload, node.materialization), - ( - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) - ) - }) - .map(|node| node.id) - .collect(); - let mut lookback: Vec<(Source, Option)> = Vec::new(); - for node in &dag.nodes { - let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload else { - continue; - }; - if node.output_state.timing != ExecutionTiming::QueryTime { - continue; - } - let scan = dag.scan_window(node.id, &kept_chains).lookback_ms(); - match lookback.iter_mut().find(|(kept, _)| kept == source) { - Some((_, kept)) => *kept = kept.zip(scan).map(|(a, b)| a.max(b)), - None => lookback.push((source.clone(), scan)), - } - } - lookback -} - -/// `rate(m[5m])` and `rate(m[5m] offset 1h)`, both at query time: the -/// runtime keeps 65 minutes of `m`, the raw lookback Stage 3 prices for the -/// same plan. With no raw data kept, both run from views refreshed at -/// ingestion and no source is read at query time. -#[tokio::test(flavor = "multi_thread")] -async fn raw_data_retention_is_the_lookback_stage3_prices() { - let workload = repeating(&["rate(m[5m])", "rate(m[5m] offset 1h)"]); - let input = || UserInput::new(&workload, FrontendInput::Promql, PlanningModels::builtin()); - let (plan, _) = plan_installable(input()).await.unwrap(); - let m = Source::TimeSeries { metric: "m".into() }; - assert_eq!( - plan.raw_data_retention().unwrap(), - [RawDataRetention::new(m.clone(), Some(65 * 60_000))] - ); - let output = e2e_plan(input()).await.unwrap(); - let selected = compile_physical_asap_workload(&output.roots()).unwrap(); - assert_eq!(stage3_raw_lookback_ms(&selected), [(m, Some(65 * 60_000))]); - - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let (plan, _) = plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .unwrap(); - assert!(!plan.materialized_views().unwrap().is_empty()); - assert_eq!(plan.raw_data_retention().unwrap(), []); -} - -/// A range-evaluated query (`max_over_time(m[5m])` over the last hour at -/// 1-minute steps) installed with every node at query time (its frontend -/// DAG, nothing materialized): its oldest step reads `m` 5 minutes before an -/// hour ago, so the runtime keeps 65 minutes of `m`, the scan's window plus -/// the range span, on the scan's node. (Stage 3 prices one evaluation's -/// window, 5 minutes.) -#[test] -fn a_range_evaluated_query_time_scan_keeps_its_window_plus_the_span() { - let workload = repeating_with( - &["max_over_time(m[5m])"], - r#", "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}"#, - ); - let roots = asap_devtools::planning_stages::lower_promql_roots(&workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let (plan, _) = installable_plan_of(&roots, None, &Default::default(), &entries).unwrap(); - plan.validate().unwrap(); - assert!(plan.materialized_views().unwrap().is_empty()); - assert_eq!( - plan.raw_data_retention().unwrap(), - [RawDataRetention::new( - Source::TimeSeries { metric: "m".into() }, - Some(5 * 60_000 + 3_600_000) - )] - ); - let scan = plan - .dag - .nodes - .iter() - .find(|node| matches!(node.payload, Operator::NonASAP(NonASAPOp::Scan { .. }))) - .unwrap(); - assert_eq!(scan.retention_ms, Some(3_900_000)); -} +use serde_json::Value; /// A plan file of wire version 10 (assembled from the parts of a plan -/// written on `main` before #970: the versioned DAG envelope, `queries`, -/// `stored_outputs`, `query_time_raw_windows`, `selection`) is refused for -/// its version, not for a missing field; a file of this version reads back. +/// written before #970: the versioned DAG envelope, `queries`, +/// `stored_outputs`, `query_time_raw_windows`, `selection`; its DAG nodes +/// carry today's node fields, e.g. `lifetime`) is refused for its version, +/// not for a missing field; its DAG with today's version reads back. #[test] fn a_plan_file_of_an_older_wire_version_is_refused_by_version() { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) diff --git a/crates/integration-tests/tests/memo_enumeration.rs b/crates/integration-tests/tests/memo_enumeration.rs deleted file mode 100644 index 6566b08e1..000000000 --- a/crates/integration-tests/tests/memo_enumeration.rs +++ /dev/null @@ -1,481 +0,0 @@ -//! The memo the production planner builds, measured on the planner-layering -//! doc's example (`avg by (job) (quantile_over_time(0.5, data[5m]) / -//! quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m]))`, -//! the stage-pipeline workload -//! `tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json`). -//! -//! Lowering is production's: the PromQL frontend, then each root with the -//! series-identity column, as `Stage1ToStage3Pass` does. The memo is read from -//! the production run's `PlanningRun::per_computation_candidates` (after -//! Pass 1) and `PlanningRun::logical_asap_candidates` (after Pass 2). -//! -//! **The memo and the whole-workload plans.** After Pass 1 the memo's -//! choices whose candidates are all sized for the whole accuracy budget -//! (`accuracy_budget_parts` 1) and the plans `compose_logical_candidate` builds from -//! the Pass 1 inventory's assignments are not equal sets: -//! - A whole-workload plan is a memo DAG exactly when targets that compute -//! the same thing take the same choice and it takes no KLL. With series -//! identity the two `quantile 0.5` are one memo group, which takes one -//! candidate; the inventory keeps a target for each and may give them -//! different summaries. The inventory sizes a KLL per value; Pass 1 sizes -//! every memo KLL for the whole result (a failure probability below the -//! query's, divided over the source's series), so no KLL plan is in both. -//! - A whole-budget memo DAG is a whole-workload plan exactly when it takes -//! no KLL sized for the whole result, that is, no KLL. -//! -//! The memo's other choices size an operand's summary for a share of the -//! budget (a ratio's operands); the whole-workload path has no such plans. -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use std::rc::Rc; - -use asap_devtools::planning_stages::lower_promql_roots; -use asap_integration_tests::planning::expanded_memo; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, PerComputationCandidates, -}; -use asap_logical_asap_optimization::{ - build_logical_asap_dags_from_chosen_candidates, enumerate, ChosenCandidatePerMemoGroup, -}; -use asap_planner::{run_planning, PlanSearchSettings, PlanningModels, PlanningRun}; -use asap_types::ir::schema::{FieldDataType, SketchParams}; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; -use asap_types::stages::{queries_rooted_at, MemoGroupId}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::workload::{PlanningWorkload, Query, RepeatedDemand, RepetitionInterval}; - -const RATIO_OF_QUANTILES: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json" -); - -/// The example's failure probability, as in its workload file. -const FAILURE_PROBABILITY: f64 = 0.001; - -/// One root per query, in query order. -type Plan = Vec>; - -fn ratio_of_quantiles() -> PlanningWorkload { - serde_json::from_str(&std::fs::read_to_string(RATIO_OF_QUANTILES).unwrap()).unwrap() -} - -/// The example's workload with `queries` in place of its query, each with -/// its requirements, repeating every `interval_ms`. -fn with_queries(queries: &[&str], interval_ms: u32) -> PlanningWorkload { - let mut workload = ratio_of_quantiles(); - let repeating = workload.query_workload.repeating_queries.as_mut().unwrap(); - let mut entry = repeating[0].clone(); - entry.demand = RepeatedDemand::FixedInterval(RepetitionInterval(interval_ms)); - *repeating = queries - .iter() - .map(|query| { - let mut entry = entry.clone(); - entry.query = Query((*query).into()); - entry - }) - .collect(); - workload -} - -/// The frontend's roots, each with the series-identity column, as -/// `Stage1ToStage3Pass` and `asap_planner` lower a PromQL workload. -fn production_roots(workload: &PlanningWorkload) -> Vec { - lower_promql_roots(workload).unwrap() -} - -/// The production run of Stages 1 to 3 over `workload`. -fn production_run(workload: &PlanningWorkload) -> PlanningRun { - production_run_with(workload, PlanningModels::builtin()) -} - -/// [`production_run`] with `models`, keeping every stage's memo. -fn production_run_with(workload: &PlanningWorkload, models: PlanningModels<'_>) -> PlanningRun { - let search = PlanSearchSettings { inspect: true }; - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let roots = production_roots(workload); - run_planning(roots, &entries, &data, models, search).unwrap() -} - -/// A deployment model that may accept an output whatever its guarantee, so -/// Pass 1 prunes no summary for its target, and accepts none: Stage 3 -/// validates no plan, and the run still records its memo. -struct AcceptsNoOutput; - -impl asap_accuracy::AccuracyModel for AcceptsNoOutput { - fn satisfies( - &self, - _guarantee: &asap_types::ir::properties::AccuracyGuarantee, - _target: &asap_types::ir::properties::AccuracyTarget, - ) -> bool { - false - } -} - -/// The memo after Pass 2. -fn pass2_memo(run: &PlanningRun) -> &Memo { - &run.logical_asap_candidates - .as_ref() - .expect("the run recorded its Pass 2 memo") - .memo -} - -/// The memo after Pass 1. -fn pass1_memo(run: &PlanningRun) -> &Memo { - &run.per_computation_candidates - .as_ref() - .expect("Pass 1's output") - .memo -} - -/// Pass 1's target list for `workload`, pruned as the facade's Pass 1 -/// prunes it under `models`. -fn pass1_inventory( - workload: &PlanningWorkload, - models: &PlanningModels<'_>, -) -> PerComputationCandidates { - let roots = production_roots(workload); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap() - .without_unsupported_summaries( - models - .deployment_capabilities - .supported_summaries - .as_deref(), - ); - let accuracy = models.accuracy_check_inputs(); - if accuracy.model.may_accept_regardless_of_guarantee() - || accuracy.evidence.may_provide_evidence() - { - return inventory; - } - let targets: Vec<_> = workload - .query_workload - .entries() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - inventory.without_summaries_no_query_can_use(&targets, &data, accuracy.model) -} - -fn candidate_count(memo: &Memo) -> usize { - memo.candidates.iter().map(Vec::len).sum() -} - -/// Every plan the whole-workload path composes from the inventory: one per -/// assignment of each target's choices, with the assignment. -fn whole_workload_plans(inventory: &PerComputationCandidates) -> Vec<(Vec, Plan)> { - let counts: Vec<_> = inventory.targets.iter().map(|t| t.choice_count()).collect(); - let total: usize = counts.iter().product(); - (0..total) - .map(|mut n| { - counts - .iter() - .map(|&count| { - let digit = n % count; - n /= count; - digit - }) - .collect::>() - }) - .filter(|choice| !inventory.absorbed_choice_varies(choice)) - .filter_map(|choice| { - let roots = compose_logical_candidate(inventory, &choice).ok()?; - let plan = roots - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator roots only"), - }) - .collect(); - Some((choice, plan)) - }) - .collect() -} - -/// `choice`'s DAG, one root per query. -fn composed(memo: &Memo, choice: &ChosenCandidatePerMemoGroup) -> Plan { - let mut roots = build_logical_asap_dags_from_chosen_candidates(memo, choice) - .expect("an enumerated choice composes"); - roots.sort_by_key(|(group, _)| { - queries_rooted_at(&memo.query_roots, *group) - .first() - .copied() - }); - roots.into_iter().map(|(_, root)| root).collect() -} - -fn chosen<'a>( - memo: &'a Memo, - choice: &'a ChosenCandidatePerMemoGroup, -) -> impl Iterator { - choice - .iter() - .map(|(group, &index)| &memo.candidates[group.0 as usize][index]) -} - -/// The KLL parameters of the summaries `candidate` builds. -fn kll(candidate: &CandidateForMemoGroup) -> Vec<(u32, Option)> { - OperatorNode::reachable(&candidate.root_node) - .iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => match kind.params() { - SketchParams::Kll { - k, - failure_probability, - } => Some((*k, *failure_probability)), - _ => None, - }, - _ => None, - }) - .collect() -} - -/// A KLL Pass 1 sized for the whole result, at the query's failure -/// probability divided over the source's series (1e-7 here), far below what -/// a per-value KLL is sized for (the query's, or half of it). -fn sized_for_whole_result(candidate: &CandidateForMemoGroup) -> bool { - kll(candidate) - .iter() - .any(|(_, p)| p.is_some_and(|p| p < FAILURE_PROBABILITY / 10.0)) -} - -/// Checks the relation of the module doc between the Pass 1 memo's -/// whole-budget DAGs and the whole-workload plans; returns (memo choices, -/// whole-budget ones, whole-workload plans, plans in both). -fn assert_pass1_memo_and_whole_workload_plans( - run: &PlanningRun, - inventory: &PerComputationCandidates, -) -> (usize, usize, usize, usize) { - let memo = pass1_memo(run); - let enumeration = enumerate(memo, 100_000); - assert!(enumeration.complete); - let plans: Vec<_> = enumeration - .all_chosen_candidates - .iter() - .map(|choice| composed(memo, choice)) - .collect(); - for (i, plan) in plans.iter().enumerate() { - assert!(!plans[..i].contains(plan), "each choice is a different DAG"); - } - let whole_budget: Vec<_> = enumeration - .all_chosen_candidates - .iter() - .zip(&plans) - .filter(|(choice, _)| chosen(memo, choice).all(|c| c.accuracy_budget_parts == 1)) - .collect(); - let whole = whole_workload_plans(inventory); - // Targets that compute the same thing. - let same: Vec<(usize, usize)> = (0..inventory.targets.len()) - .flat_map(|a| (a + 1..inventory.targets.len()).map(move |b| (a, b))) - .filter(|&(a, b)| { - let (a, b) = (&inventory.targets[a], &inventory.targets[b]); - inventory.aggregate(a) == inventory.aggregate(b) && a.measure_index == b.measure_index - }) - .collect(); - let takes_kll = |plan: &Plan| { - plan.iter().flat_map(OperatorNode::reachable).any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if matches!(kind.params(), SketchParams::Kll { .. })) - }) - }; - for (assignment, plan) in &whole { - assert_eq!( - whole_budget.iter().any(|(_, p)| *p == plan), - same.iter().all(|&(a, b)| assignment[a] == assignment[b]) && !takes_kll(plan), - "a whole-workload plan is a memo DAG when equal targets choose alike \ - and it takes no (per-value) KLL" - ); - assert!( - !plans.iter().any(|p| p == plan) || whole_budget.iter().any(|(_, p)| *p == plan), - "a whole-workload plan is never a budget-share DAG" - ); - } - for (choice, plan) in &whole_budget { - assert_eq!( - whole.iter().any(|(_, p)| p == *plan), - !chosen(memo, choice).any(sized_for_whole_result), - "a whole-budget memo DAG is a whole-workload plan unless it takes a \ - KLL sized for the whole result" - ); - } - let both = whole - .iter() - .filter(|(_, plan)| whole_budget.iter().any(|(_, p)| p == &plan)) - .count(); - (plans.len(), whole_budget.len(), whole.len(), both) -} - -/// The example's memo after each production step, and the `accuracy_budget_parts` -/// of the state-configuration rule's rewrites (one test: one planning run). -#[test] -fn ratio_of_quantiles_production_memo() { - // With the built-in model, Pass 1 prunes every summary of the quantile - // groups: no query has a guarantee through `/` and `+` over a KLL rank - // error or a DDSketch relative one. The memo below is planned with a - // model that prunes none and accepts no output, so its summaries stay - // and its Pass 2 memo is recorded. - let enforced = production_run(&ratio_of_quantiles()); - assert_eq!(candidate_count(pass1_memo(&enforced)), 7); - let models = PlanningModels::builtin().with_accuracy(&AcceptsNoOutput); - let run = production_run_with(&ratio_of_quantiles(), models); - // Stage 0 (Pass 1's groups): series identity makes the two - // `quantile 0.5` one group. - // Pass 1: the two quantile groups have 5 candidates each, 3 for the - // whole budget (exact, DDSketch, KLL sized for the whole result) and 2 - // for half of it; the other 5 groups one each. - assert_eq!(pass1_memo(&run).groups.len(), 7); - assert_eq!(candidate_count(pass1_memo(&run)), 15); - // 5 × 5 choices, 3 × 3 of them for the whole budget. The inventory has - // 3 × 3 assignments (one target per quantile group and measure, so the - // two `quantile 0.5` share one, 3 choices each): the 2 × 2 that take no - // KLL are memo DAGs. - assert_eq!( - assert_pass1_memo_and_whole_workload_plans( - &run, - &pass1_inventory(&ratio_of_quantiles(), &models) - ), - (25, 9, 9, 4) - ); - // The last step, over the complete tumbling-window-width domain. - let last = pass2_memo(&run); - assert_eq!(last.groups.len(), 255); - assert_eq!(candidate_count(last), 1243); - let enumeration = enumerate(last, 2_000); - assert!(!enumeration.complete); - assert_eq!(enumeration.all_chosen_candidates.len(), 2_000); - for (group, candidates) in last.candidates.iter().enumerate() { - for index in 0..candidates.len() { - let choice = [(MemoGroupId(group as u32), index)].into(); - build_logical_asap_dags_from_chosen_candidates(last, &choice) - .unwrap_or_else(|e| panic!("G{group} candidate {index}: {e}")); - } - } - // The state-configuration rule rewrites a candidate's KLL to the size - // the readers of a state share; the rewrite is a copy of the candidate - // (`derive` in `pass2/memo/rules/shared_configuration_readers.rs`) with - // the `accuracy_budget_parts` its new size was made for. In each quantile - // group the `accuracy_budget_parts` 1 KLL `k` 567 (sized for the whole - // result) gets copies at `k` 567 (parts 1) and 1184 (parts 2, the size of - // the half-budget KLL); the `accuracy_budget_parts` 2 KLL `k` 1184 keeps - // 2. The `k` 1184 copy composes the same DAG as the original - // `accuracy_budget_parts` 2 candidate: one sketch, sized for half the - // budget. - let pass1 = pass1_memo(&run); - let quantile_groups: Vec<_> = (0..pass1.groups.len()) - .filter(|&group| pass1.candidates[group].len() == 5) - .collect(); - assert_eq!(quantile_groups.len(), 2); - for group in quantile_groups { - let sizes = |candidates: &[CandidateForMemoGroup]| { - candidates - .iter() - .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Pass1) - .filter_map(|c| kll(c).first().map(|&(k, _)| (k, c.accuracy_budget_parts))) - .collect::>() - }; - assert_eq!(sizes(&pass1.candidates[group]), [(567, 1), (1184, 2)]); - let candidates = &last.candidates[group]; - assert_eq!( - sizes(candidates), - [(567, 1), (1184, 2), (567, 1), (1184, 2)] - ); - let at_k: Vec = candidates - .iter() - .enumerate() - .filter(|(_, c)| { - c.origin == OriginOfCandidateForMemoGroup::Pass1 - && kll(c).first().is_some_and(|&(k, _)| k == 1184) - }) - .map(|(index, _)| index) - .collect(); - assert_eq!(at_k.len(), 2); - let alone = |index: usize| composed(last, &[(MemoGroupId(group as u32), index)].into()); - assert_eq!(alone(at_k[0]), alone(at_k[1])); - } -} - -/// On the three-query workload, the same relation: the memo's 3 × 3 -/// whole-budget DAGs and the inventory's 3 × 3 assignments share the 2 × 2 -/// that take no KLL (the memo's is sized for the whole result, the -/// inventory's per value), with the default search. -#[test] -fn shared_quantiles_pass1_memo_and_whole_workload_plans() { - let workload = with_queries( - &[ - "quantile_over_time(0.5, data[5m])", - "quantile_over_time(0.9, data[5m])", - "sum by (job) (rate(data[1m]))", - ], - 60_000, - ); - let run = production_run(&workload); - assert_eq!( - assert_pass1_memo_and_whole_workload_plans( - &run, - &pass1_inventory(&workload, &PlanningModels::builtin()) - ), - (9, 9, 9, 4) - ); -} - -/// Pass 2 rejects a memo whose groups read each other, a group reading -/// itself included, so production's memo (coarse and fine quantiles, whose -/// rollups read the finer grouping) has a dependency order and every choice -/// `enumerate` returns composes. A cycle made by hand, a child candidate -/// reading back the group that reads it, composes no DAG: -/// `build_logical_asap_dags_from_chosen_candidates` says so. The memo is -/// built from production's roots on the seed grids, without the search. -#[test] -fn choices_whose_candidates_read_each_other_are_left_out() { - let workload = with_queries( - &[ - "quantile_over_time(0.5, data[1m])", - "quantile_over_time(0.9, data[5m])", - "quantile by (host, job) (0.5, data)", - "quantile by (job) (0.5, data)", - ], - 10_000, - ); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let memo = expanded_memo(production_roots(&workload), &entries); - assert!(memo.topological_order().is_some()); - for (group, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates { - assert!(!candidate.input_groups.contains(&MemoGroupId(group as u32))); - } - } - let enumeration = enumerate(&memo, 2_000); - assert!(!enumeration.all_chosen_candidates.is_empty()); - for choice in &enumeration.all_chosen_candidates { - build_logical_asap_dags_from_chosen_candidates(&memo, choice).unwrap(); - } - - let (group, index, child) = (0..memo.groups.len()) - .flat_map(|group| (0..memo.candidates[group].len()).map(move |index| (group, index))) - .find_map(|(group, index)| { - memo.candidates[group][index] - .input_groups - .first() - .map(|&child| (MemoGroupId(group as u32), index, child)) - }) - .expect("a candidate reading another group"); - let mut cyclic = memo.clone(); - let mut back = cyclic.candidates[child.0 as usize][0].clone(); - back.root_node = Rc::clone(&cyclic.groups[group.0 as usize].logical_node); - back.input_groups = vec![group]; - cyclic.candidates[child.0 as usize].push(back); - let back_index = cyclic.candidates[child.0 as usize].len() - 1; - assert_eq!(cyclic.topological_order(), None); - assert!(matches!( - build_logical_asap_dags_from_chosen_candidates( - &cyclic, - &[(group, index), (child, back_index)].into() - ), - Err(asap_logical_asap_optimization::ErrorBuildingLogicalASAPDagsFromChosenCandidates::Cycle { .. }) - )); -} diff --git a/crates/integration-tests/tests/nested.rs b/crates/integration-tests/tests/nested.rs index bcaaaf9ed..52c235341 100644 --- a/crates/integration-tests/tests/nested.rs +++ b/crates/integration-tests/tests/nested.rs @@ -13,11 +13,11 @@ use std::time::Duration; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::fixtures::metric_schema; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::{ AggIntent, AtModifier, BinaryOpKind, GroupKeys, PromQLVectorSetOpKind, Reduction, Source, TimeShift, VectorMatch, VectorMatchKind, }; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::{ BinaryOperator, ExprSemantics, NonASAPOp, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, diff --git a/crates/integration-tests/tests/operator_design_examples.rs b/crates/integration-tests/tests/operator_design_examples.rs index 88b19291d..80ceb6abf 100644 --- a/crates/integration-tests/tests/operator_design_examples.rs +++ b/crates/integration-tests/tests/operator_design_examples.rs @@ -1,14 +1,12 @@ //! #511 examples: source text → unified dag → summary rewrite → flat export. -use crate::physical_common; + use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{ - ExactKind, ExactParams, FieldDataType, GroupingStrategy, SummaryUpdate, -}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; +use asap_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; use std::rc::Rc; fn catalog() -> SqlCatalog { @@ -32,7 +30,6 @@ async fn sql_filter_projection_example() { let root = lower_sql( "SELECT l_quantity * 2 AS q2 FROM lineitem WHERE l_quantity > 10", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap(); @@ -44,7 +41,7 @@ async fn sql_filter_projection_example() { assert!( matches!(root.expect_non_asap(),NonASAPOp::Project { cols,.. } if matches!(cols[0].expr,ScalarExpr::Arithmetic { .. })) ); - let wire = physical_common::compile_physical_asap_dag(&root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); wire.validate().unwrap(); assert_eq!(wire.nodes.len(), OperatorNode::reachable(&root).len()); } @@ -55,7 +52,6 @@ async fn sql_sum_projection_before_and_after_summary_rewrite() { let root = lower_sql( "SELECT SUM(bytes) + 1 AS total_bytes FROM requests WHERE status = 200", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap(); @@ -78,12 +74,11 @@ async fn sql_sum_projection_before_and_after_summary_rewrite() { let state = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: Rc::clone(child), - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), input: SummaryUpdate::column(ColumnRef::Named( child.schema.fields[*column].name.clone(), )), reduction: reduction.clone(), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap(), @@ -111,7 +106,7 @@ async fn sql_sum_projection_before_and_after_summary_rewrite() { assert!(dag .iter() .any(|n| matches!(n.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))); - let wire = physical_common::compile_physical_asap_dag(&rewritten).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&rewritten).unwrap(); wire.validate().unwrap(); assert_eq!(wire.nodes.len(), dag.len()); } @@ -123,16 +118,14 @@ async fn sql_scalar_subquery_retains_its_cardinality_contract() { "SELECT (SELECT bytes FROM requests) AS v FROM lineitem", "SELECT l_quantity NOT IN (SELECT bytes FROM requests) AS present FROM lineitem", ] { - let root = lower_sql(query, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let root = lower_sql(query, &catalog()).await.unwrap(); root.validate_structure().unwrap(); assert!(root.children().len() > 1); - let wire = physical_common::compile_physical_asap_dag(&root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); assert!(wire .edges .iter() - .any(|e| e.role == asap_types::ir::physical_export::EdgeRole::ScalarRef)); + .any(|e| e.role == asap_types::plan::physical_asap_dag::EdgeRole::ScalarRef)); } } @@ -150,7 +143,6 @@ async fn sql_sum_example_executes_with_sql_null_semantics() { let root = lower_sql( "SELECT SUM(bytes) + 1 AS total_bytes FROM requests WHERE status = 200", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap(); @@ -161,14 +153,14 @@ async fn sql_sum_example_executes_with_sql_null_semantics() { let NonASAPOp::Scan { source, .. } = logical_scan.expect_non_asap() else { panic!() }; - let wire = physical_common::compile_physical_asap_dag(&root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); let scan = wire .nodes .iter() .find(|node| { matches!( &node.payload, - asap_types::ir::physical_export::PhysicalASAPOperatorPayload::NonASAP( + asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload::NonASAP( asap_types::ir::NonASAPOp::Scan { .. } ) ) @@ -231,151 +223,8 @@ async fn sql_window_and_filtered_aggregate_types() { "SELECT MIN(l_quantity) OVER (ORDER BY l_quantity ROWS BETWEEN 2 PRECEDING AND 1 PRECEDING) AS s FROM lineitem", "SELECT SUM(l_quantity) FILTER (WHERE l_quantity < 0) AS s FROM lineitem GROUP BY l_quantity", ] { - let root = lower_sql(query, &catalog(), AccuracyTarget::exact()).await.unwrap(); + let root = lower_sql(query, &catalog()).await.unwrap(); root.validate_structure().unwrap(); assert_eq!(root.schema.fields[0], Field::plain("s", DataType::Int64, true), "{query}"); } } - -/// A real query batch replaces each SUM by its exact accumulator state, -/// retains two result roots, and executes both selected plans. Pass 2 -/// shares the identical state, and building the shared sub-DAG once per -/// evaluation for both queries (`MatAtQuery`, on the state or on its -/// finalized sum above it) is the cheapest plan, so both roots read one -/// state. No replacement dag is constructed by the test. -#[tokio::test] -async fn batch_planning_replaces_summary_operators() { - use asap_executor::values::Value; - use asap_planner::PlanningModels; - use asap_planner::{e2e_plan, FrontendInput, UserInput}; - use asap_types::workload::*; - let queries = [ - "SELECT SUM(bytes) + 1 AS result FROM requests", - "SELECT SUM(bytes) * 2 AS result FROM requests", - ]; - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), - query_batch: Some( - queries - .iter() - .map(|query| BatchEntry { - query: Query((*query).into()), - requirements: QueryRequirements { - accuracy: AccuracyTarget::exact(), - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 2, - execute_at: None, - range_evaluation: None, - }) - .collect(), - ), - repeating_queries: None, - }, - // `requests` at rest, holding a million rows. - data_workload: Some(DataWorkload { - arrival: DataArrival::AtRest, - sources: [( - "requests".to_string(), - asap_types::workload::SourceFacts { - rows_held: Some(1_000_000), - ..Default::default() - }, - )] - .into(), - }), - }; - let catalog = SqlCatalog::new().with_table( - "requests", - Schema::new(vec![Field::plain("bytes", DataType::Float64, false)]), - ); - let output = e2e_plan(UserInput::new( - &workload, - FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin(), - )) - .await - .unwrap(); - assert_eq!(output.entry_indices(), [0, 1]); - assert_eq!(output.roots().len(), 2); - let states: Vec<_> = output - .operators() - .into_iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect(); - let [state] = &states[..] else { - panic!("one SUM state for both queries: {states:?}") - }; - assert!(output.operators().into_iter().any(|n| { - matches!( - n.materialization, - asap_types::physical::Materialization::MatAtQuery(_) - ) && OperatorNode::reachable(&n) - .iter() - .any(|m| Rc::ptr_eq(m, state)) - })); - for (plan, expected) in output.plans.iter().zip([31.0, 60.0]) { - let root = &plan.root; - root.validate_structure().unwrap(); - assert!(OperatorNode::reachable(root) - .iter() - .any(|n| Rc::ptr_eq(n, state))); - // The table `requests` bound to its two rows. - let rows = physical_common::execute_raw_rows( - root, - vec![vec![Value::Float64(10.0)], vec![Value::Float64(20.0)]], - ); - assert_eq!(rows.len(), 1); - assert!( - matches!(rows[0][0], Value::Float64(v) if v == expected), - "{:?}", - rows - ); - } - // The batch exports as one physical DAG: a root per query, the one SUM - // state, and the shared node the selected plan builds once. - let workload_dag = output.physical_asap_dag().unwrap(); - assert_eq!(workload_dag.roots.len(), 2); - assert_ne!(workload_dag.roots[0], workload_dag.roots[1]); - let states: Vec<_> = workload_dag - .nodes - .iter() - .filter(|n| { - matches!( - n.payload, - asap_types::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - asap_types::ir::ASAPOp::SummaryAgg { .. } - ) - ) - }) - .collect(); - let [state] = &states[..] else { - panic!("one SUM state: {states:?}") - }; - // The node built once is the state or a node above it. - let reaches_state = |from| { - let mut stack = vec![from]; - let mut seen = std::collections::HashSet::new(); - while let Some(id) = stack.pop() { - if id == state.id { - return true; - } - if seen.insert(id) { - stack.extend( - workload_dag - .edges - .iter() - .filter(|e| e.consumer == id) - .map(|e| e.producer), - ); - } - } - false - }; - assert!(workload_dag.nodes.iter().any(|n| matches!( - n.materialization, - asap_types::physical::Materialization::MatAtQuery(_) - ) && reaches_state(n.id))); -} diff --git a/crates/integration-tests/tests/operator_sharing.rs b/crates/integration-tests/tests/operator_sharing.rs index 0d1d07345..22dec7ea8 100644 --- a/crates/integration-tests/tests/operator_sharing.rs +++ b/crates/integration-tests/tests/operator_sharing.rs @@ -11,7 +11,7 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::physical_export::physical_asap_dag; use asap_integration_tests::planning::selected_dag; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; @@ -47,7 +47,7 @@ fn catalog() -> SqlCatalog { /// Lower `sql` and take the logical DAG the planner pipeline selects. async fn plan(sql: &str, accuracy: AccuracyTarget) -> Rc { - let pre = lower_sql(sql, &catalog(), accuracy.clone()) + let pre = lower_sql(sql, &catalog()) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); selected_dag(pre, accuracy) diff --git a/crates/integration-tests/tests/per_computation_candidates_sql.rs b/crates/integration-tests/tests/per_computation_candidates_sql.rs index 6902cd681..b62e18c8e 100644 --- a/crates/integration-tests/tests/per_computation_candidates_sql.rs +++ b/crates/integration-tests/tests/per_computation_candidates_sql.rs @@ -3,15 +3,11 @@ //! ranks. use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, -}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::ColumnRef; +use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SummaryInputExpr}; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; -use asap_types::stages::{ComputationTarget, HavingPlacement, HowToComputeAggregateMeasure}; +use asap_types::ir::QueryRoot; +use asap_types::stages::{ComputationTarget, HavingPlacement}; async fn targets(sql: &str) -> Vec { let catalog = SqlCatalog::new().with_table( @@ -21,14 +17,16 @@ async fn targets(sql: &str) -> Vec { Field::plain("bytes", DataType::Float64, false), ]), ); - let accuracy = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let root = lower_sql(sql, &catalog, accuracy) + let root = lower_sql(sql, &catalog) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap() .targets @@ -70,16 +68,16 @@ async fn mixed_key_and_value_having_stays_above() { /// Whether `node` reads a quantile sketch (KLL or DDSketch) anywhere below it. fn reads_quantile_sketch(node: &std::rc::Rc) -> bool { - use asap_types::ir::schema::SketchAlgorithm; + use asap_types::summary_type::SketchAlgorithm; asap_types::ir::OperatorNode::reachable(node) .iter() .any(|node| { matches!( node.asap(), Some(asap_types::ir::ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(kind, _), + summary_type: asap_types::ir::schema::FieldDataType::Sketch(kind), .. - }) if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch) + }) if matches!(kind, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch) ) }) } @@ -123,7 +121,7 @@ async fn a_filter_or_group_key_over_an_estimate_has_no_guarantee() { "SELECT count(p) FROM \ (SELECT src, approx_percentile_cont(bytes, 0.99) AS p FROM flows GROUP BY src) s", ] { - let root = lower_sql(sql, &catalog, accuracy.clone()) + let root = lower_sql(sql, &catalog) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); // No guarantee meets the target, so Pass 1 prunes the sketch @@ -142,198 +140,3 @@ async fn a_filter_or_group_key_over_an_estimate_has_no_guarantee() { } } } - -/// Whether `node` reads a DDSketch anywhere below it. -fn reads_ddsketch(node: &std::rc::Rc) -> bool { - use asap_types::ir::schema::SketchAlgorithm; - asap_types::ir::OperatorNode::reachable(node) - .iter() - .any(|node| { - matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(kind, _), - .. - }) if matches!(kind.algorithm(), SketchAlgorithm::DDSketch) - ) - }) -} - -/// A SQL fold over the groups of an approximate subquery counts rows the -/// series count of its source does not bound, so its δ names its own -/// row-count statistic (`group_row_count`), not the source's -/// `input_row_count`. -#[tokio::test] -async fn a_sql_fold_over_estimates_leaves_its_row_count_unbound() { - let catalog = SqlCatalog::new().with_table( - "flows", - Schema::new(vec![ - Field::plain("src", DataType::Utf8, false), - Field::plain("bytes", DataType::Float64, false), - ]), - ); - let accuracy = AccuracyTarget::max_error(0.01); - // q=1 is an endpoint, so this isolates the fold's row-count statistic - // without assuming a sign for SQL bytes or interpolating across zero. - let sql = "SELECT max(p) FROM \ - (SELECT src, approx_percentile_cont(bytes, 1.0) AS p FROM flows GROUP BY src) s"; - let root = lower_sql(sql, &catalog, accuracy.clone()).await.unwrap(); - let candidates = vec![ - asap_integration_tests::planning::explicit_summary_candidate( - root, - SketchAlgorithm::DDSketch, - ), - ]; - assert!(candidates.iter().all(reads_ddsketch)); - assert!(!candidates.is_empty()); - for candidate in candidates { - let g = candidate.guarantee.clone().expect("a relative max"); - let delta = format!("{:?}", g.spec.failure_probability); - assert!(delta.contains("group_row_count"), "{delta}"); - assert!(!delta.contains("input_row_count"), "{delta}"); - } -} - -/// The item and weight columns of every heap sketch build in each composed -/// candidate of `sql` over `flows(key: key_type, bytes)` that picks a heap -/// sketch for its top-k. -async fn heap_updates(key: &str, key_type: DataType, sql: &str) -> Vec<(String, String)> { - heap_updates_over( - vec![ - Field::plain(key, key_type, false), - Field::plain("bytes", DataType::Float64, false), - ], - sql, - ) - .await -} - -/// [`heap_updates`] over `flows(fields)`. -async fn heap_updates_over(fields: Vec, sql: &str) -> Vec<(String, String)> { - let catalog = SqlCatalog::new().with_table("flows", Schema::new(fields)); - let accuracy = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let name = |expr: &SummaryInputExpr| match expr { - SummaryInputExpr::Column(ColumnRef::Named(name)) => name.clone(), - SummaryInputExpr::Column(ColumnRef::Qualified { table, name }) => { - assert_eq!(table, "flows"); - name.clone() - } - // The ranked value when it is the child's one sample value. - SummaryInputExpr::Column(ColumnRef::SampleValue) => "".into(), - SummaryInputExpr::Tuple(items) => match items.as_slice() { - [SummaryInputExpr::Column(ColumnRef::Named(name))] => name.clone(), - [SummaryInputExpr::Column(ColumnRef::Qualified { table, name })] => { - assert_eq!(table, "flows"); - name.clone() - } - other => panic!("one item column: {other:?}"), - }, - SummaryInputExpr::Constant(value) => value.to_string(), - other => panic!("a named column or constant: {other:?}"), - }; - let mut updates = Vec::new(); - for (t, target) in inventory.targets.iter().enumerate() { - for c in 0..target.choice_count() { - let (candidate, _) = target.computation_for(c).unwrap(); - if !matches!( - candidate, - HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap - ) - ) { - continue; - } - let mut choice = vec![0; inventory.targets.len()]; - choice[t] = c; - let composed = compose_logical_candidate(&inventory, &choice) - .unwrap_or_else(|e| panic!("{candidate:?} composes: {e}")); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - for node in OperatorNode::reachable(root) { - if let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - input, - .. - }) = node.asap() - { - updates.push((name(input.item.as_ref().unwrap()), name(&input.weight))); - } - } - } - } - updates -} - -const COUNT_RANKED_TOPK: &str = - "SELECT KEY, count(*) FROM flows GROUP BY KEY ORDER BY count(*) DESC LIMIT 10"; - -/// An Int64 key is not mistaken for the ranked value: both heap sketches -/// compose over raw rows, keyed by the key with unit weight. -#[tokio::test] -async fn topk_heap_over_an_int64_key_counts_raw_rows() { - let updates = heap_updates( - "port", - DataType::Int64, - &COUNT_RANKED_TOPK.replace("KEY", "port"), - ) - .await; - assert_eq!(updates.len(), 2, "{updates:?}"); - assert!( - updates - .iter() - .all(|(item, weight)| item == "port" && weight == "1"), - "{updates:?}" - ); -} - -/// A key named `value` is not the ranked value: the heap sketches are -/// fed raw rows with unit weight. -#[tokio::test] -async fn topk_heap_over_a_key_named_value_counts_raw_rows() { - let updates = heap_updates( - "value", - DataType::Utf8, - &COUNT_RANKED_TOPK.replace("KEY", "value"), - ) - .await; - assert_eq!(updates.len(), 2, "{updates:?}"); - assert!( - updates - .iter() - .all(|(item, weight)| item == "value" && weight == "1"), - "{updates:?}" - ); -} - -/// SQL's per-group top-k idiom (`ROW_NUMBER() OVER (PARTITION BY city -/// ORDER BY COUNT(*) DESC) … WHERE rn <= 5`) is a heavy-hitter top-k, as -/// PromQL `topk by (city)` is: both heap sketches compose, keyed by the -/// host and fed raw rows with unit weight. -#[tokio::test] -async fn rownumber_per_group_topk_offers_heap_sketches() { - let updates = heap_updates_over( - vec![ - Field::plain("city", DataType::Utf8, false), - Field::plain("host", DataType::Utf8, false), - ], - "SELECT city, host, c FROM (SELECT city, host, COUNT(*) AS c, ROW_NUMBER() OVER \ - (PARTITION BY city ORDER BY COUNT(*) DESC) AS rn FROM flows GROUP BY city, host) \ - WHERE rn <= 5", - ) - .await; - assert_eq!(updates.len(), 2, "{updates:?}"); - assert!( - updates - .iter() - .all(|(item, weight)| item == "host" && weight == "1"), - "{updates:?}" - ); -} diff --git a/crates/integration-tests/tests/per_series_exact_execution.rs b/crates/integration-tests/tests/per_series_exact_execution.rs index e20fed25b..7ff4e94e0 100644 --- a/crates/integration-tests/tests/per_series_exact_execution.rs +++ b/crates/integration-tests/tests/per_series_exact_execution.rs @@ -13,14 +13,11 @@ use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; use asap_integration_tests::fixtures::lower_promql; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::{Reduction, Source}; -use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOperatorPayload}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{Schema, PROMQL_SERIES_IDENTITY}; -use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, NonASAPOp, - OperatorNode, -}; +use asap_types::ir::{NonASAPOp, OperatorNode}; +use asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; use futures::{executor::block_on, StreamExt}; /// One raw data sample: every label of its series, its time and value. @@ -96,13 +93,8 @@ fn series_labels(schema: &Schema, row: &[Value]) -> BTreeMap { /// Run `query`'s exact per-entity plan at query time over `samples`, bound /// at its `Scan` of `x`: each answered series' `instance` and value. fn execute(query: &str, samples: &[RawDataSample]) -> Vec<(String, f64)> { - let timed = apply_materialization_timings( - &per_entity_plan(query), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let plan = per_entity_plan(query); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(&plan).unwrap(); let node = |matches: fn(&PhysicalASAPOperatorPayload) -> bool| { let found: Vec<_> = dag .nodes diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index e9f16cfa9..84181645c 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -33,40 +33,6 @@ pub fn execute( }) } -#[allow(dead_code)] -pub fn compile_physical_asap_dag( - root: &std::rc::Rc, -) -> Result> { - compile_with(root, &asap_types::ir::MaterializationAssignment::default()) -} - -/// As [`compile_physical_asap_dag`], with every sub-DAG `MatAtIngestion` -/// where the DAG allows it. -#[allow(dead_code)] // Not every test binary sharing this module stores states at ingestion. -pub fn compile_at_ingestion_physical_asap_dag( - root: &std::rc::Rc, -) -> Result> { - compile_with( - root, - &asap_types::ir::MaterializationAssignment::with_default( - asap_types::physical::Materialization::MatAtIngestion( - asap_types::physical::Medium::Memory, - ), - ), - ) -} - -fn compile_with( - root: &std::rc::Rc, - assignment: &asap_types::ir::MaterializationAssignment, -) -> Result> { - let root = - asap_types::ir::apply_materialization_timings(root, assignment, &mut Default::default())?; - Ok(asap_types::ir::physical_export::compile_physical_asap_dag( - &root, - )?) -} - /// Execute a relational DAG at query time against an in-memory connector /// for its one scan, scan predicates included; checks that no memory stays /// reserved afterwards. @@ -87,9 +53,9 @@ pub fn execute_raw_rows_at( evaluation_time_ms: i64, ) -> Vec> { use asap_executor::physical_planner::compile; - use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::NonASAPOp; - let wire = compile_physical_asap_dag(root).unwrap(); + use asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; + let wire = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); let (source, schema) = wire .nodes .iter() @@ -127,14 +93,14 @@ pub fn execute_raw_rows_at( /// `rows(source, schema)` in that `Scan`'s schema. #[allow(dead_code)] pub fn raw_data_store( - dag: &asap_types::ir::physical_export::PhysicalASAPDAG, + dag: &asap_types::plan::physical_asap_dag::PhysicalASAPDAG, mut rows: impl FnMut( &asap_types::ir::operator::Source, &asap_executor::values::SchemaRef, ) -> Vec>, ) -> InMemoryDataStore { - use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::NonASAPOp; + use asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; let mut store = InMemoryDataStore::default(); let mut bound = Vec::new(); for node in &dag.nodes { diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index 47357ac32..ca8b332db 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -17,42 +17,20 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use asap_cost::CostInputs; use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_plan_selection::reference_pricer::WholePlanSelection as Selection; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::flat::{flatten, FlatDag, NodeId}; -use asap_types::ir::physical_export::PhysicalASAPDAG; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot}; -use asap_types::physical::Materialization; -use asap_types::physical::PhysicalWorkloadPlan; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; +use asap_types::stages::Lifetime; +use asap_types::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; use asap_types::workload::{PlanningWorkload, QueryRecurrence, RepeatedDemand}; pub use asap_devtools::planner_layering_examples::*; pub type Payload = Operator; -/// Lower and plan a PromQL workload. -pub fn run_promql(workload: &PlanningWorkload) -> Run { - run_stages(workload, lower_promql(workload)) -} - -/// Lower and plan a PromQL workload, selecting under `settings` (a cost -/// model other than the default). -pub fn run_promql_under(workload: &PlanningWorkload, settings: &CostInputs) -> Run { - run_stages_under(workload, lower_promql(workload), settings) -} - -/// Lower and plan a PromQL workload for a deployment with `deployment`'s -/// capabilities, under the default cost model. -pub fn run_promql_for(workload: &PlanningWorkload, deployment: &DeploymentCapabilities) -> Run { - run_stages_with( - workload, - lower_promql(workload), - &CostInputs::default(), - deployment, - ) -} - /// Whether `form` answers a query window by merging several summaries. pub fn needs_merge(form: WindowForm, query_window_ms: u64) -> bool { match form { @@ -62,25 +40,6 @@ pub fn needs_merge(form: WindowForm, query_window_ms: u64) -> bool { } } -/// `roots` with every identical sub-DAG made one node (the identical- -/// expression rule, as the memo's group identity applies it). -#[allow(dead_code)] -pub fn share_identical(roots: &[QueryRoot]) -> Vec { - asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, std::rc::Rc::clone(node)), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect() -} - /// `roots` with no node shared between queries: each query reads its own /// copy of every node it reaches (sharing within one query is kept). Pass 1 /// composes a candidate from memo groups, so queries that read an identical @@ -114,22 +73,6 @@ pub fn unshare(roots: &[QueryRoot]) -> Vec { .collect() } -/// The number of distinct nodes (by identity) `roots` reach: smaller after -/// [`share_identical`] exactly when some sub-DAG was shared. -#[allow(dead_code)] -pub fn distinct_nodes(roots: &[QueryRoot]) -> usize { - roots - .iter() - .filter_map(|root| match root { - QueryRoot::Operator(node) => Some(node), - QueryRoot::Scalar(_) => None, - }) - .flat_map(asap_types::ir::OperatorNode::reachable) - .map(|node| std::rc::Rc::as_ptr(&node)) - .collect::>() - .len() -} - // ── Query-language frontends ────────────────────────────────────────── /// PromQL workload roots, each series' identity as a column. @@ -184,48 +127,12 @@ pub struct Physical { pub id: String, pub from_logical: String, pub query_roots: Vec, - pub candidate: PhysicalWorkloadPlan, + pub dag: PhysicalASAPDAG, } impl Physical { pub fn dag(&self) -> &PhysicalASAPDAG { - &self.candidate.dag - } -} - -#[derive(Debug, Clone)] -pub struct Run { - pub stage0_query_language_frontends: Logical, - pub logical: Vec, - pub physical: Vec, - /// Logical candidates physical ASAP-aware optimization cannot implement: - /// (logical id, reason). - pub pruned: Vec<(String, String)>, - pub selection: Selection, -} - -impl Run { - pub fn invalid(&self) -> BTreeMap<&str, &str> { - self.selection - .rejected - .iter() - .filter(|r| r.cause.failed_a_check()) - .map(|r| (r.id.as_str(), r.reason.as_str())) - .collect() - } - - pub fn cost(&self, id: &str) -> Option { - self.selection.costs.get(id).and_then(|c| c.cost_per_second) - } - - pub fn physical(&self, id: &str) -> &Physical { - self.physical.iter().find(|p| p.id == id).expect("id") - } - - pub fn physical_of<'a>(&'a self, logical: &'a Logical) -> impl Iterator { - self.physical - .iter() - .filter(move |p| p.from_logical == logical.id) + &self.dag } } @@ -247,212 +154,6 @@ fn logical(id: String, roots: Vec) -> Logical { } } -/// The bounded production stages over `roots`, retaining the selected plan -/// and search diagnostics. Tests that require a particular algorithm or window -/// use `run_explicit_candidates` instead of assuming it appears in the archive. -pub fn run_stages(workload: &PlanningWorkload, roots: Vec) -> Run { - run_stages_under(workload, roots, &CostInputs::default()) -} - -/// [`run_stages`], selecting under `settings`. -pub fn run_stages_under( - workload: &PlanningWorkload, - roots: Vec, - settings: &CostInputs, -) -> Run { - run_stages_with( - workload, - roots, - settings, - &DeploymentCapabilities::default(), - ) -} - -/// [`run_stages`], selecting under `settings` for a deployment with -/// `deployment`'s capabilities. -pub fn run_stages_with( - workload: &PlanningWorkload, - roots: Vec, - settings: &CostInputs, - deployment: &DeploymentCapabilities, -) -> Run { - let models = asap_planner::PlanningModels::builtin() - .with_cost_inputs(settings) - .with_deployment_capabilities(deployment); - run_stages_over(workload, roots, models) -} - -fn run_stages_over( - workload: &PlanningWorkload, - roots: Vec, - models: asap_planner::PlanningModels<'_>, -) -> Run { - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - // The facade's run, with the reference pricer over the Stage 2 - // memo choices: the whole-plan costs and rejections these tests read. - let run = run_planning_stages_over(roots, &entries, &data, None, models) - .expect("planning stages run"); - let reference = run.reference.expect("the memo is enumerated"); - let logical_candidates = reference - .choices - .into_iter() - .map(|c| { - let roots = c.roots.unwrap_or_else(|e| panic!("{} composes: {e}", c.id)); - logical(c.id, roots) - }) - .collect(); - let physical = reference - .physical - .into_iter() - .map(|candidate| Physical { - id: candidate.id.clone(), - from_logical: candidate.from_logical.clone(), - query_roots: candidate.dag.roots.clone(), - candidate, - }) - .collect(); - Run { - stage0_query_language_frontends: logical( - "stage0-query-language-frontends".into(), - run.stage0_query_language_frontends, - ), - logical: logical_candidates, - physical, - pruned: reference.stage2_failed, - selection: reference.selection.expect("plan selection selects"), - } -} - -/// Evaluate deliberately supplied logical configurations. This fixture helper -/// does not search or enumerate cross-query combinations; it exercises the -/// existing physical rules and evaluator on the computations named by a test. -/// Production-search tests continue to use `run_stages`. -pub fn run_explicit_candidates( - workload: &PlanningWorkload, - original_roots: Vec, - candidates: Vec>, - models: asap_planner::PlanningModels<'_>, -) -> Run { - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - // Explicit candidates are composed outside the memo: their summaries are - // sized as Pass 1 sizes a candidate, which Stage 3 only checks. - let logical: Vec<_> = candidates - .into_iter() - .enumerate() - .map(|(index, roots)| { - let roots = asap_accuracy::propagation::recompose_guarantees(&roots) - .expect("explicit fixture contracts compose"); - logical( - format!("P{}", index + 1), - asap_accuracy::sizing::with_summaries_sized_for_targets(&roots, &targets, &data) - .expect("explicit fixture summaries rebuild"), - ) - }) - .collect(); - let mut physical = Vec::new(); - for candidate in &logical { - for (index, mut plan) in asap_physical_asap_optimization::optimize_physical_asap( - &candidate.id, - &candidate.roots, - &entries, - &data, - models.deployment_capabilities.query_time_results_retained, - ) - .expect("explicit computation has executable physical configurations") - .into_iter() - .enumerate() - { - plan.id = if index == 0 { - candidate.id.clone() - } else { - format!("{}-m{}", candidate.id, index) - }; - physical.push(Physical { - id: plan.id.clone(), - from_logical: plan.from_logical.clone(), - query_roots: plan.dag.roots.clone(), - candidate: plan, - }); - } - } - // Stage 2's deployment rules (raw data retention) prune the plans the - // deployment cannot run before Stage 3 prices the rest. - let mut plans: Vec<_> = physical.iter().map(|plan| plan.candidate.clone()).collect(); - asap_devtools::memo_pricing::retain_deployable(&mut plans, models); - physical.retain(|p| plans.iter().any(|plan| plan.id == p.id)); - let selection = asap_devtools::memo_pricing::reference_select(&plans, &entries, &data, models) - .expect("explicit fixture contains a valid physical configuration"); - Run { - stage0_query_language_frontends: logical_stage0(original_roots), - logical, - physical, - pruned: Vec::new(), - selection, - } -} - -pub fn run_pattern_b_fixture( - workload: &PlanningWorkload, - settings: &CostInputs, - deployment: &DeploymentCapabilities, -) -> Run { - run_explicit_candidates( - workload, - lower_promql(workload), - pattern_b_computations(workload), - asap_planner::PlanningModels::builtin() - .with_cost_inputs(settings) - .with_deployment_capabilities(deployment), - ) -} - -/// Three fixed Pattern-B computations used by cost/retention fixtures: -/// exact, whole-window KLL, and a KLL chain of sixty-second tumbling windows. -pub fn pattern_b_computations(workload: &PlanningWorkload) -> Vec> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let roots = lower_promql(workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - assert_eq!( - local.targets.len(), - 1, - "Pattern B has one quantile computation" - ); - let kll = local.targets[0] - .ways_to_compute - .iter() - .position(|candidate| { - matches!( - candidate, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap(); - let whole: Vec<_> = compose_logical_candidate(&local, &[kll]).unwrap(); - let tumbling_windows = tumbling_window_candidates(&whole, &[Some(60_000)]); - assert_eq!(tumbling_windows.len(), 1); - let chain = window_composition_pass2_candidates(&whole, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .expect("sixty-second tumbling windows compose"); - vec![roots, whole, chain] -} - fn logical_stage0(roots: Vec) -> Logical { logical("stage0-query-language-frontends".into(), roots) } @@ -528,7 +229,7 @@ pub fn closure(dag: &impl ExportedDag, root: NodeId) -> HashSet { } /// The queries (indexes into `query_roots`) whose result depends on `node`. -pub fn readers(dag: &impl ExportedDag, query_roots: &[NodeId], node: NodeId) -> BTreeSet { +pub fn queries(dag: &impl ExportedDag, query_roots: &[NodeId], node: NodeId) -> BTreeSet { query_roots .iter() .enumerate() @@ -561,10 +262,10 @@ pub fn is_summary(payload: &Payload) -> bool { payload, Operator::ASAP( ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. } | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. } | ASAPOp::SummaryEstimate { .. } | ASAPOp::SummaryMerge { .. } @@ -576,27 +277,6 @@ pub fn is_merge(payload: &Payload) -> bool { matches!(payload, Operator::ASAP(ASAPOp::SummaryMerge { .. })) } -/// The sketch build nodes (`SummaryAgg` or tumbling-window `SummaryTumblingWindowChain` over a -/// sketch family) of `dag`, with their algorithm and parameters. -pub fn sketch_builds(dag: &impl ExportedDag) -> Vec<(NodeId, SketchAlgorithm, SketchParams)> { - dag.node_ids() - .into_iter() - .filter_map(|id| match dag.payload(id) { - Operator::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => Some((id, *kind.algorithm(), kind.params().clone())), - _ => None, - }) - .collect() -} - /// The estimation nodes that read `build`, directly or through merges. pub fn estimates_of(dag: &impl ExportedDag, build: NodeId) -> Vec<(NodeId, SketchStatistic)> { let mut out = Vec::new(); @@ -619,168 +299,26 @@ pub fn estimates_of(dag: &impl ExportedDag, build: NodeId) -> Vec<(NodeId, Sketc out } -/// One query's per-computation choice: the sorted sketch algorithms built in -/// its closure, or `"exact"` when it builds none. -pub fn query_option(dag: &impl ExportedDag, query_roots: &[NodeId], query: usize) -> String { - let reach = closure(dag, query_roots[query]); - let mut names: Vec<_> = sketch_builds(dag) - .into_iter() - .filter(|(id, ..)| reach.contains(id)) - .map(|(_, algorithm, _)| format!("{algorithm:?}")) - .collect(); - names.sort(); - if names.is_empty() { - "exact".into() - } else { - names.join("+") - } -} - /// Nodes reachable from more than one query root. pub fn cross_query_nodes(dag: &impl ExportedDag, query_roots: &[NodeId]) -> BTreeSet { dag.node_ids() .into_iter() - .filter(|&id| readers(dag, query_roots, id).len() > 1) + .filter(|&id| queries(dag, query_roots, id).len() > 1) .collect() } -pub fn assert_valid_and_uniquely_named(run: &Run) { - let ids: BTreeSet<_> = run.logical.iter().map(|c| c.id.as_str()).collect(); - assert_eq!(ids.len(), run.logical.len(), "unique logical ids"); - for c in &run.logical { - for root in &c.roots { - root.validate_structure() - .unwrap_or_else(|e| panic!("{}: {e}", c.id)); - } - } -} - -/// No candidate is discarded before plan selection: physical ASAP-aware -/// optimization gives each logical candidate `L` exactly one physical -/// candidate with nothing materialized, `P`, and one `P-m` for each -/// other materialization assignment. -pub fn assert_one_unmaterialized_physical_candidate_per_logical(run: &Run) { - assert!(run.pruned.is_empty(), "pruned: {:?}", run.pruned); - let unmaterialized: Vec<_> = run - .physical - .iter() - .filter(|p| !p.id.contains("-m")) - .collect(); - let sources: BTreeSet<_> = unmaterialized - .iter() - .map(|p| p.from_logical.as_str()) - .collect(); - let logical: BTreeSet<_> = run.logical.iter().map(|c| c.id.as_str()).collect(); - assert_eq!(sources, logical); - assert_eq!(unmaterialized.len(), run.logical.len()); - for p in &run.physical { - let base = p.id.split("-m").next().unwrap(); - assert_eq!(&base[1..], &p.from_logical[1..], "{}", p.id); - let unmaterialized = !p.id.contains("-m"); - assert_eq!(unmaterialized, !p.candidate.materializes_any(), "{}", p.id); - if unmaterialized { - for n in &p.dag().nodes { - assert_eq!( - materialization(p, n.id), - MaterializationOption::NotMaterialized - ); - } - } - } - let ids: BTreeSet<_> = run.physical.iter().map(|p| p.id.as_str()).collect(); - assert_eq!(ids.len(), run.physical.len(), "unique physical ids"); -} - -/// The physical candidates with nothing materialized, one per logical -/// candidate. -pub fn unmaterialized(run: &Run) -> impl Iterator { - run.physical - .iter() - .filter(|p| !p.candidate.materializes_any()) -} - -/// One selected id; every other candidate rejected once, with a reason. -pub fn assert_selects_one_and_explains_the_rest(run: &Run) { - let all: BTreeSet<_> = run.physical.iter().map(|p| p.id.clone()).collect(); - let mut accounted: BTreeSet<_> = run - .selection - .rejected - .iter() - .map(|r| r.id.clone()) - .collect(); - assert_eq!( - accounted.len(), - run.selection.rejected.len(), - "rejected once each" - ); - assert!(run.selection.rejected.iter().all(|r| !r.reason.is_empty())); - assert!( - accounted.insert(run.selection.selected.clone()), - "selected is not rejected" - ); - assert_eq!(accounted, all); -} - -/// The selected plan costs no more than any valid candidate. -pub fn assert_selects_cheapest_valid(run: &Run) { - let invalid = run.invalid(); - let best = run - .cost(&run.selection.selected) - .expect("selected is priced"); - for p in run - .physical - .iter() - .filter(|p| !invalid.contains_key(p.id.as_str())) - { - // Totals within rounding of each other tie (plan selection's rule). - let cost = run.cost(&p.id).unwrap(); - assert!(best <= cost + cost.abs() * 1e-9, "{} is cheaper", p.id); - } -} - -/// Each valid candidate's cost has one entry per DAG node, `total` is their -/// sum, and each node's multiplier is [`computations`]. Invalid candidates -/// are not priced. -pub fn assert_each_node_charged_per_consuming_query(run: &Run) { - let invalid = run.invalid(); - for p in &run.physical { - if invalid.contains_key(p.id.as_str()) { - assert!(run.cost(&p.id).is_none(), "{} is priced", p.id); - continue; - } - let cost = &run.selection.costs[&p.id]; - let nodes: BTreeSet<_> = p.dag().nodes.iter().map(|n| n.id).collect(); - let charged: BTreeSet<_> = cost.per_node.keys().copied().collect(); - assert_eq!(charged, nodes, "{}", p.id); - let sum: f64 = cost - .per_node - .values() - .map(|c| c.cost_per_second.unwrap()) - .sum(); - assert!( - (cost.cost_per_second.unwrap() - sum).abs() <= 1e-9 * sum.abs().max(1.0), - "{}", - p.id - ); - let times = computations(p.dag()); - for (id, c) in &cost.per_node { - assert_eq!(c.multiplier, times[id], "{}: multiplier of {id:?}", p.id); - } - } -} - /// How many times each node is computed per evaluation of its consumers (#509; the /// rule of plan selection's private `computations`): 1 for a materialized -/// node; for a `NotMaterialized` one, the number of consumers that reach +/// node; for a not-materialized one, the number of consumers that reach /// it. A consumer is a query (operator or scalar root) or a materialized /// node, whose walk starts at its inputs; a walk stops at materialized -/// nodes and counts each node once. The `MatAtIngestion` consumers of one +/// nodes and counts each node once. The consumers materialized at ingestion time of one /// refresh window share one ingestion run and count once (#1151). pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { - let materialized: BTreeSet<_> = dag + let materialized_nodes: BTreeSet<_> = dag .nodes .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) + .filter(|n| n.lifetime != asap_types::stages::Lifetime::None) .map(|n| n.id) .collect(); // Each walk's start and, for a consumer refreshed at ingestion, its @@ -794,22 +332,22 @@ pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { .iter() .map(|(_, expr)| (expr.operator_refs().into_iter().copied().collect(), None)), ) - .chain(materialized.iter().map(|&id| { - let at_ingestion = matches!( - dag.nodes - .iter() - .find(|n| n.id == id) - .unwrap() - .materialization, - Materialization::MatAtIngestion(_) - ); + .chain(materialized_nodes.iter().map(|&id| { + let at_ingestion = dag + .nodes + .iter() + .find(|n| n.id == id) + .unwrap() + .output_state + .timing + == ExecutionTiming::IngestionTime; let window = at_ingestion.then(|| dag.refresh_window(id)).flatten(); (dag.producers(id), window) })); let mut times: BTreeMap<_, _> = dag .nodes .iter() - .map(|n| (n.id, u64::from(materialized.contains(&n.id)))) + .map(|n| (n.id, u64::from(materialized_nodes.contains(&n.id)))) .collect(); // The (node, refresh window) pairs whose ingestion run is counted. let mut runs = HashSet::new(); @@ -817,7 +355,7 @@ pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { let mut seen = HashSet::new(); let mut stack = start; while let Some(id) = stack.pop() { - if materialized.contains(&id) || !seen.insert(id) { + if materialized_nodes.contains(&id) || !seen.insert(id) { continue; } if window.is_none_or(|window| runs.insert((id, window))) { @@ -838,32 +376,30 @@ pub fn evaluation_interval_ms(workload: &PlanningWorkload, query: usize) -> Opti } } -// ── Materialization ────────────────────────────────────────────────────── +// ── Materialized outputs ───────────────────────────────────────────────────────── -/// A node's materialization option (#509 "Materialization"), without its -/// storage medium. +/// Whether a node's output is materialized and at which timing, without how long +/// (`OneEvaluation` or `AcrossEvaluations`) or its storage medium. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum MaterializationOption { - /// Runs as data arrives; output stored before any query asks. - MatAtIngestion, +pub enum Materializing { + /// Runs as data arrives; output materialized until a query reads it. + MaterializedAtIngestion, /// Runs when a query first needs it; output reused by the other - /// queries of the evaluation, or, as `MatAtQueryKeptAcrossEvaluations`, - /// kept for later evaluations. - MatAtQuery, - /// Runs at query time for each execution; output discarded. + /// queries of the evaluation, or, as query-time `AcrossEvaluations`, + /// retained for later evaluations. + MaterializedAtQueryTime, + /// Output discarded after its query (lifetime `None`). NotMaterialized, } -/// The materialization option physical ASAP-aware optimization assigned to -/// `node` (#667). -pub fn materialization(p: &Physical, node: NodeId) -> MaterializationOption { +/// Whether physical ASAP-aware optimization materializes `node`'s output, and at +/// which timing (#667). +pub fn materializing(p: &Physical, node: NodeId) -> Materializing { let n = p.dag().nodes.iter().find(|n| n.id == node).expect("node"); - match n.materialization { - Materialization::MatAtIngestion(_) => MaterializationOption::MatAtIngestion, - Materialization::MatAtQuery(_) | Materialization::MatAtQueryKeptAcrossEvaluations(_) => { - MaterializationOption::MatAtQuery - } - Materialization::NotMaterialized => MaterializationOption::NotMaterialized, + match (n.output_state.timing, n.lifetime) { + (_, Lifetime::None) => Materializing::NotMaterialized, + (ExecutionTiming::IngestionTime, _) => Materializing::MaterializedAtIngestion, + (ExecutionTiming::QueryTime, _) => Materializing::MaterializedAtQueryTime, } } @@ -872,25 +408,6 @@ pub fn runs_at_ingestion(p: &Physical, node: NodeId) -> bool { n.output_state.timing == asap_types::ir::properties::ExecutionTiming::IngestionTime } -/// Every node upstream of an ingestion-time node also runs at ingestion time. -pub fn assert_ingestion_upstream_is_ingestion(run: &Run) { - for p in &run.physical { - for n in &p.dag().nodes { - if !runs_at_ingestion(p, n.id) { - continue; - } - for up in closure(p.dag(), n.id) { - assert!( - runs_at_ingestion(p, up), - "{}: {up:?} feeds ingestion-time {:?} at query time", - p.id, - n.id - ); - } - } - } -} - // ── Pending adapters ───────────────────────────────────────────────────── /// The window summary a summary build node maintains (#509 Pass 2, @@ -922,49 +439,3 @@ pub fn window_form(dag: &impl ExportedDag, build: NodeId) -> WindowForm { _ => WindowForm::None, } } - -/// How far back from an evaluation a materialized node's kept output -/// reaches in event time, derived the way plan selection prices what it -/// keeps: a tumbling-window chain keeps its `N` -/// tumbling windows of width `w`, plus the one being built when maintained at -/// ingestion time; a whole-window build keeps its window, and the next one -/// being built at ingestion time; each plus the offset of the time shifts -/// below it. `None` for any other node. -pub fn retention_ms(p: &Physical, node: NodeId) -> Option { - let dag = p.dag(); - let at_ingestion = runs_at_ingestion(p, node); - let below = |id: NodeId| dag.producers(id).into_iter().next(); - let offset_below = |mut id: Option| { - let mut offset = 0u64; - while let Some(at) = id { - match dag.payload(at) { - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { - offset += shift.offset_ms.unsigned_abs() - } - Operator::NonASAP(NonASAPOp::Scan { .. }) => break, - _ => {} - } - id = below(at); - } - offset - }; - match dag.payload(node) { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => { - let tumbling_windows = u64::from(*tumbling_window_count) + u64::from(at_ingestion); - Some(tumbling_windows * tumbling_window_width_ms + offset_below(below(node))) - } - Operator::ASAP(ASAPOp::SummaryAgg { .. }) => { - let child = below(node)?; - let Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) = dag.payload(child) else { - return None; - }; - let windows = 1 + u64::from(at_ingestion); - Some(windows * range.as_millis() as u64 + offset_below(below(child))) - } - _ => None, - } -} diff --git a/crates/integration-tests/tests/planner_layering_example1.rs b/crates/integration-tests/tests/planner_layering_example1.rs deleted file mode 100644 index ca983c655..000000000 --- a/crates/integration-tests/tests/planner_layering_example1.rs +++ /dev/null @@ -1,951 +0,0 @@ -//! Rule-fixture and production acceptance for Example 1: aggregation across dimensions. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::planner_layering_common; - -use std::collections::{BTreeMap, BTreeSet, HashSet}; - -use asap_devtools::memo_pricing::reference_select as select_plan; -use asap_planner::PlanningModels; -use asap_types::ir::flat::NodeId; -use asap_types::ir::schema::{FieldDataType, GroupingStrategy, HydraKind, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot, ScalarExpr, SortKey}; -use asap_types::physical::{Materialization, Medium}; -use planner_layering_common::*; - -// ── Example 1 workload ─────────────────────────────────────────────────── - -fn pipeline() -> Run { - rule_fixture(&example1()) -} - -// Explicit rule fixtures: vary Q2's three documented algorithms while Q1 -// stays exact, and exercise source sharing and ten-second tumbling windows. This is not -// a production workload search or an exhaustive memo archive. -fn rule_fixture(workload: &asap_types::workload::PlanningWorkload) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let roots = lower_promql(workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut computations = Vec::new(); - for family in [ - None, - Some(SketchAlgorithm::CmsWithHeap), - Some(SketchAlgorithm::CountSketchWithHeap), - ] { - let mut choice = vec![0; local.targets.len()]; - if let Some(family) = family { - let (target, option) = local - .targets - .iter() - .enumerate() - .find_map(|(target, choices)| { - (0..choices.choice_count()) - .find(|&option| { - matches!(choices.computation_for(option), - Some((HowToComputeAggregateMeasure::Summary(algorithm), _)) if *algorithm == family) - }) - .map(|option| (target, option)) - }) - .expect("documented heap family offered locally"); - choice[target] = option; - } - let whole = unshare(&compose_logical_candidate(&local, &choice).unwrap()); - let mut windows = vec![whole.clone()]; - let tumbling_windows = tumbling_window_candidates(&whole, &[Some(10_000), Some(10_000)]); - windows.extend( - window_composition_pass2_candidates(&whole, &tumbling_windows) - .unwrap() - .into_iter() - .map(|candidate| candidate.roots.unwrap()), - ); - // Each window form alone and with its identical input shared. The - // memo shares an identical range selector whole, never its scan - // alone; that variant reads the same rows twice and is dominated. - for window in windows { - let shared = share_identical(&window); - let sharing = distinct_nodes(&shared) < distinct_nodes(&window); - computations.push(window); - if sharing { - computations.push(shared); - } - } - } - run_explicit_candidates(workload, roots, computations, PlanningModels::builtin()) -} - -#[test] -fn production_memo_selects_an_executable_accurate_example1_plan() { - let workload = example1(); - let run = run_stages_under(&workload, lower_promql(&workload), &Default::default()); - assert_selects_cheapest_valid(&run); - compile_in_runtime(run.physical(&run.selection.selected)).unwrap(); -} - -// ── Helpers ────────────────────────────────────────────────────────────── - -/// Q2's per-computation option for its top-k, read off its summary build node. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -enum Q2Option { - Exact, - CountMinHeapPerJob, - CountSketchHeapPerJob, - Hydra, -} - -const PASS1_Q2_OPTIONS: [Q2Option; 3] = [ - Q2Option::Exact, - Q2Option::CountMinHeapPerJob, - Q2Option::CountSketchHeapPerJob, -]; - -/// Sketch families built in `nodes`, as Example 1's Q2 options. -fn sketch_options(dag: &impl ExportedDag, nodes: &HashSet) -> BTreeSet { - nodes - .iter() - .filter_map(|&id| match dag.payload(id) { - Operator::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, grouping), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, grouping), - .. - }, - ) => Some(match (grouping, kind.algorithm()) { - ( - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - .. - }, - _, - ) => Q2Option::Hydra, - (GroupingStrategy::PerSubpopulationInstance, SketchAlgorithm::CmsWithHeap) => { - Q2Option::CountMinHeapPerJob - } - ( - GroupingStrategy::PerSubpopulationInstance, - SketchAlgorithm::CountSketchWithHeap, - ) => Q2Option::CountSketchHeapPerJob, - (other, algorithm) => { - panic!("summary family outside Example 1: {algorithm:?} {other:?}") - } - }), - _ => None, - }) - .collect() -} - -/// Whether every summary of `dag` is built over its whole window: no -/// tumbling windows (Pass 2's window-composition rule). -fn whole_windows(dag: &impl ExportedDag) -> bool { - dag.node_ids().into_iter().all(|id| { - !matches!( - dag.payload(id), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) -} - -fn roots(query_roots: &[NodeId]) -> (NodeId, NodeId) { - assert_eq!(query_roots.len(), 2, "every candidate covers Q1 and Q2"); - (query_roots[0], query_roots[1]) -} - -/// Q2's option and whether Q1 and Q2 share any node, for one candidate. -fn classify(dag: &impl ExportedDag, query_roots: &[NodeId]) -> (Q2Option, bool) { - let (q1, q2) = roots(query_roots); - let (c1, c2) = (closure(dag, q1), closure(dag, q2)); - let options = sketch_options(dag, &c2); - assert!(options.len() <= 1, "Q2 uses one option: {options:?}"); - let option = options.into_iter().next().unwrap_or(Q2Option::Exact); - (option, !c1.is_disjoint(&c2)) -} - -/// The candidate with Q2 `option` whose queries share no node and whose -/// summaries are built over whole windows (Pass 2 adds shared-input and -/// tumbling-window variants of each). -fn physical_with(run: &Run, option: Q2Option) -> &Physical { - let mut found = run - .physical - .iter() - .filter(|p| whole_windows(p.dag()) && classify(p.dag(), &p.query_roots) == (option, false)); - let p = found.next().unwrap_or_else(|| panic!("no {option:?}")); - assert!(found.next().is_none(), "one {option:?} candidate"); - p -} - -/// Example 1's panels repeat every 10 s. -fn every_10s() -> asap_types::workload::QueryRecurrence { - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(10_000))) -} - -/// The workload entry of an exact query evaluated every 10 s. -fn exact_every_10s() -> asap_types::workload::QueryWorkloadEntry { - asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: Default::default(), - predictability: Default::default(), - recurrence: every_10s(), - range_evaluation: None, - } -} - -// ── Workload ───────────────────────────────────────────────────────────── - -/// The encoded workload is valid and normalizes to Q1 then Q2. -#[test] -fn workload_encodes_example1() { - let workload = example1(); - workload.validate().expect("valid workload"); - let entries: Vec<_> = workload.query_workload.entries().collect(); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].query.0, EXAMPLE1_Q1); - assert_eq!(entries[1].query.0, EXAMPLE1_Q2); -} - -// ── Query-language frontends ────────────────────────────────────────── - -/// The PromQL frontend lowers each query to the doc's chain. -#[test] -fn stage0_query_language_frontends_lower_each_query_to_doc_chain() { - let roots = lower_promql(&example1()); - let chains: Vec<_> = roots.iter().map(frontend_operations).collect(); - assert_eq!( - chains, - [ - ["aggregate:rate", "aggregate:sum", "scan", "time_range"], - ["aggregate:sum", "aggregate:top_k", "scan", "time_range"], - ] - ); -} - -/// The frontends yield one workload DAG with one root per query and no summaries. -#[test] -fn stage0_query_language_frontends_one_summary_free_workload_dag() { - let stage0_query_language_frontends = pipeline().stage0_query_language_frontends; - roots(&stage0_query_language_frontends.query_roots); - assert!(stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); -} - -/// The frontends keep Q1 and Q2 separate; sharing is a logical ASAP-aware optimization decision. -#[test] -fn stage0_query_language_frontends_queries_do_not_share_nodes() { - let stage0_query_language_frontends = pipeline().stage0_query_language_frontends; - assert!(!stage0_query_language_frontends.shared_input()); -} - -// ── Logical ASAP-aware optimization ────────────────────────────────────── - -/// Pass 1 gives 1 × 3 = 3 candidates without a shared input: Q2's top-k is -/// exact, Count-Min + heap or CountSketch + heap. (Pass 2 adds shared-input -/// and tumbling-window variants of each.) -#[test] -fn stage1_logical_asap_optimization_has_three_pass1_candidates() { - let run = pipeline(); - let found: Vec<_> = run - .logical - .iter() - .filter(|c| whole_windows(&c.dag)) - .map(|c| classify(&c.dag, &c.query_roots)) - .filter(|(_, shared)| !shared) - .collect(); - let expected: Vec<_> = PASS1_Q2_OPTIONS.into_iter().map(|o| (o, false)).collect(); - let mut sorted = found.clone(); - sorted.sort(); - assert_eq!(sorted, expected, "each Q2 option exactly once: {found:?}"); -} - -/// Q1 is exact in every candidate: no sketch build or estimate is reachable -/// from its root (its rate may be merged from exact tumbling windows). -#[test] -fn stage1_logical_asap_optimization_q1_is_always_exact() { - let run = pipeline(); - for c in &run.logical { - let (q1, _) = roots(&c.query_roots); - assert!( - closure(&c.dag, q1).iter().all(|&id| { - let payload = c.dag.payload(id); - !is_summary(payload) || is_merge(payload) - }), - "{}: Q1 reaches a summary", - c.id - ); - } -} - -/// Every Pass 1 target's exact option is its exact accumulator: Q1's rate -/// and sum and Q2's sum_over_time. Q2's heaps are whole-expression: they -/// absorb its sum_over_time, which is then not computed. -#[test] -fn stage1_logical_asap_optimization_exact_targets_use_accumulators() { - let run = pipeline(); - for c in &run.logical { - let expected = match classify(&c.dag, &c.query_roots).0 { - Q2Option::Exact => 3, - _ => 2, - }; - let accumulators = c - .dag - .nodes - .iter() - .filter(|n| { - matches!( - n.operator, - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) - ) - }) - .count(); - assert_eq!(accumulators, expected, "{}", c.id); - } -} - -/// Q2's summary families are Count-Min + heap and CountSketch + heap, one per job. -#[test] -fn stage1_logical_asap_optimization_q2_summary_families() { - let run = pipeline(); - let families: BTreeSet<_> = run - .logical - .iter() - .flat_map(|c| { - let (_, q2) = roots(&c.query_roots); - sketch_options(&c.dag, &closure(&c.dag, q2)) - }) - .collect(); - assert_eq!( - families, - BTreeSet::from([ - Q2Option::CountMinHeapPerJob, - Q2Option::CountSketchHeapPerJob - ]) - ); -} - -/// Pass 1 also offers Hydra (HydraCms over all (job, series) keys, a heap per job) for Q2's top-k. -#[test] -#[ignore = "needs Pass 1 Hydra for grouped approximate counts (#703)"] -fn stage1_logical_asap_optimization_q2_offers_hydra() { - let run = pipeline(); - assert!(run - .logical - .iter() - .any(|c| classify(&c.dag, &c.query_roots).0 == Q2Option::Hydra)); -} - -/// Pass 2's identical-expression rule adds a shared-input variant of every Q2 option and keeps the independent one. -#[test] -fn stage1_logical_asap_optimization_keeps_independent_and_shared_variants() { - let run = pipeline(); - let found: Vec<_> = run - .logical - .iter() - .map(|c| classify(&c.dag, &c.query_roots)) - .collect(); - for option in PASS1_Q2_OPTIONS { - assert!(found.contains(&(option, false)), "{option:?} independent"); - assert!(found.contains(&(option, true)), "{option:?} shared"); - } -} - -/// Pass 2's window-composition rule builds Q1's rate and Q2's option in six -/// 10-s tumbling windows (the gcd of the 1-min window and the 10-s interval), -/// each alone and both together, next to the whole-window candidates: 27 -/// candidates in all. -#[test] -fn stage1_logical_asap_optimization_window_composition_adds_10s_tumbling_windows() { - let run = pipeline(); - let mut chains = BTreeSet::new(); - for c in &run.logical { - for id in c.dag.node_ids() { - if let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - tumbling_window_count, - .. - }) = c.dag.payload(id) - { - let state = match family { - FieldDataType::ExactAggregate(kind, _) => format!("{kind:?}"), - FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), - other => panic!("{}: {other:?}", c.id), - }; - let query = readers(&c.dag, &c.query_roots, id); - chains.insert(( - state, - query, - *tumbling_window_width_ms, - *tumbling_window_count, - )); - } - } - } - let expected: BTreeSet<_> = [ - ("Rate", 0), - ("Sum", 1), - ("CmsWithHeap", 1), - ("CountSketchWithHeap", 1), - ] - .into_iter() - .map(|(state, query)| (state.to_string(), BTreeSet::from([query]), 10_000, 6)) - .collect(); - assert_eq!(chains, expected); - let whole = run.logical.iter().filter(|c| whole_windows(&c.dag)).count(); - assert!(whole > 0); -} - -/// Only the raw input is ever shared between Q1 and Q2; no summary is shared. -#[test] -fn stage1_logical_asap_optimization_shares_input_but_never_a_summary() { - let run = pipeline(); - for c in &run.logical { - for id in cross_query_nodes(&c.dag, &c.query_roots) { - let p = c.dag.payload(id); - assert!( - matches!(relational(p).as_deref(), Some("scan" | "time_range")), - "{}: shared node {id:?} is not the range selector input: {p:?}", - c.id - ); - } - } -} - -/// Logical candidates are valid DAGs with unique ids. -#[test] -fn stage1_logical_asap_optimization_candidates_are_valid_and_uniquely_named() { - assert_valid_and_uniquely_named(&pipeline()); -} - -// ── Physical ASAP-aware optimization ───────────────────────────────────── - -/// No candidate is discarded before plan selection: each logical candidate -/// keeps one physical candidate with nothing materialized, 3 of them -/// without a shared input or tumbling windows. -#[test] -fn stage2_physical_asap_optimization_keeps_every_logical_candidate() { - let run = pipeline(); - assert_one_unmaterialized_physical_candidate_per_logical(&run); - let unshared = unmaterialized(&run) - .filter(|p| whole_windows(p.dag()) && !classify(p.dag(), &p.query_roots).1) - .count(); - assert_eq!(unshared, 3); -} - -/// Each physical candidate keeps its logical Q2 option, and one with -/// nothing materialized keeps its input sharing. Maintaining Q2's tumbling windows at -/// ingestion time splits a scan Q1 reads at query time, one copy per phase. -#[test] -fn stage2_physical_asap_optimization_preserves_logical_choices() { - let run = pipeline(); - for p in &run.physical { - let source = run.logical.iter().find(|c| c.id == p.from_logical).unwrap(); - let (option, shared) = classify(p.dag(), &p.query_roots); - let (logical_option, logical_shared) = classify(&source.dag, &source.query_roots); - assert_eq!(option, logical_option, "{} vs {}", p.id, source.id); - if !p.candidate.materializes_any() { - assert_eq!(shared, logical_shared, "{} vs {}", p.id, source.id); - } - } -} - -/// Exact TopK is implemented as a sort followed by a limit. -#[test] -fn stage2_physical_asap_optimization_exact_topk_is_sort_then_limit() { - let run = pipeline(); - let p = physical_with(&run, Q2Option::Exact); - let sort_then_limit = p.dag().edges.iter().any(|e| { - relational(p.dag().payload(e.producer)).as_deref() == Some("sort") - && relational(p.dag().payload(e.consumer)).as_deref() == Some("limit") - }); - assert!(sort_then_limit, "{}: no sort → limit", p.id); -} - -/// Exact Q2 sorts each job's series by the ranked value, descending, and a -/// `Project` over the limit restores the logical `[job, topk_10]` columns, -/// then keeps each chosen series' identity (its full label set). -#[test] -fn stage2_physical_asap_optimization_exact_topk_returns_job_and_topk_10() { - let run = pipeline(); - let p = physical_with(&run, Q2Option::Exact); - let QueryRoot::Operator(root) = &p.candidate.roots[1] else { - panic!("{}: Q2 is an operator root", p.id) - }; - let names: Vec<_> = root.schema.fields.iter().map(|f| f.name.as_str()).collect(); - assert_eq!( - names, - ["job", "topk_10", "$promql_series_identity"], - "{}", - p.id - ); - let NonASAPOp::Project { child: limit, .. } = root.expect_non_asap() else { - panic!("{}: Q2's root is a Project", p.id) - }; - let NonASAPOp::Limit { - n: Some(10), - child: sort, - .. - } = limit.expect_non_asap() - else { - panic!("{}: a limit 10 under the Project", p.id) - }; - let NonASAPOp::Sort { - keys, - partition_by, - child, - } = sort.expect_non_asap() - else { - panic!("{}: a sort under the limit", p.id) - }; - let [SortKey { - expr: ScalarExpr::Column(ranked), - ascending: false, - .. - }] = keys.as_slice() - else { - panic!("{}: one descending column key: {keys:?}", p.id) - }; - assert_eq!(child.schema.fields[*ranked].name, "value", "{}", p.id); - assert_eq!(partition_by.len(), 1, "{}: partitioned by job", p.id); -} - -/// A summary Q2 over its whole window is a build node feeding a top-10 -/// estimation node, with no merge. -#[test] -fn stage2_physical_asap_optimization_summary_topk_is_build_then_estimate() { - let run = pipeline(); - for p in &run.physical { - if classify(p.dag(), &p.query_roots).0 == Q2Option::Exact || !whole_windows(p.dag()) { - continue; - } - assert!( - !p.dag().nodes.iter().any(|n| is_merge(&n.payload)), - "{}: no window summaries, so no merge", - p.id - ); - let build_to_estimate = p.dag().edges.iter().any(|e| { - matches!( - p.dag().payload(e.producer), - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) && matches!( - p.dag().payload(e.consumer), - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - ) - }); - assert!(build_to_estimate, "{}: no build → estimate", p.id); - } -} - -/// Only tumbling windows can be maintained at ingestion time: the -/// whole-window builds' fixed interval has no phase, and no summary is read -/// by both queries. Q1's rate tumbling windows are counter states, which need each -/// series' samples in time order; continuously ingesting PromQL data is -/// taken to arrive in that order, so they are maintained too. Each tumbling window -/// chain is maintained or not independently: a logical candidate with `c` -/// chains has `2^c` physical candidates. A shared input (the highest node -/// both queries read: the scan, or the time range above it) is also built -/// once per evaluation (`MatAtQuery`), which adds one candidate: the one -/// with every chain at query time, since a chain maintained at ingestion -/// time leaves one query reading the input at query time. -#[test] -fn stage2_physical_asap_optimization_materializes_only_tumbling_windows_at_ingestion() { - let run = pipeline(); - let mut counter_maintained = 0; - let mut shared_once = 0; - for p in &run.physical { - for n in &p.dag().nodes { - match materialization(p, n.id) { - MaterializationOption::NotMaterialized => {} - MaterializationOption::MatAtIngestion => { - let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { family, .. }) = - p.dag().payload(n.id) - else { - panic!("{}: {:?} is not a tumbling-window chain", p.id, n.id) - }; - if matches!( - family, - FieldDataType::ExactAggregate( - asap_types::ir::schema::ExactKind::Rate - | asap_types::ir::schema::ExactKind::Increase, - _ - ) - ) { - counter_maintained += 1; - } - } - MaterializationOption::MatAtQuery => { - let (q1, q2) = roots(&p.query_roots); - assert!( - closure(p.dag(), q1).contains(&n.id) - && closure(p.dag(), q2).contains(&n.id), - "{}: {:?} is built once for both queries", - p.id, - n.id - ); - assert!( - matches!( - relational(p.dag().payload(n.id)).as_deref(), - Some("scan" | "time_range") - ), - "{}: {:?} is the shared input", - p.id, - n.id - ); - shared_once += 1; - } - } - } - } - assert!( - counter_maintained > 0, - "Q1's rate tumbling windows are maintained" - ); - assert!( - shared_once > 0, - "explicit shared-source fixtures are materialized" - ); - let expected: usize = run - .logical - .iter() - .map(|c| { - let chains = c - .dag - .node_ids() - .into_iter() - .filter(|&id| { - matches!( - c.dag.payload(id), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .count(); - (1 << chains) + usize::from(c.shared_input()) - }) - .sum(); - assert_eq!(run.physical.len(), expected); -} - -/// Compile `p` in the physical planner (the runtime capability check), with -/// its materialized views as view slots and every `Scan` as raw data. -fn compile_in_runtime(p: &Physical) -> Result<(), String> { - use asap_executor::physical_planner::compile; - let mut dag = p.dag().clone(); - let views = dag - .set_retention(&Default::default(), &Default::default()) - .map_err(|e| format!("{}: {e}", p.id))?; - let roots: Vec = dag.roots.iter().map(|&id| id as u64).collect(); - compile(&dag, &views, &roots) - .map(|_| ()) - .map_err(|e| format!("{}: {e}", p.id)) -} - -// ── Plan selection ─────────────────────────────────────────────────────── - -/// Plan selection selects one candidate and gives every other one a reason. -#[test] -fn stage3_plan_selection_selects_one_and_explains_the_rest() { - assert_selects_one_and_explains_the_rest(&pipeline()); -} - -/// The exact Q2 is selected. Both heaps are invalid because a top-k output -/// has no guarantee and the deployment has no evidence for it. Count-Min's -/// weights are raw data samples of `http_requests_total`, which the workload -/// declares a counter, so they are non-negative. -#[test] -fn stage3_plan_selection_selects_exact_q2_and_rejects_both_heaps() { - let run = pipeline(); - let selected = run.physical(&run.selection.selected); - assert_eq!( - classify(selected.dag(), &selected.query_roots).0, - Q2Option::Exact - ); - let invalid = run.invalid(); - // Only heap candidates are invalid (with or without a shared input). - for id in invalid.keys() { - let p = run.physical(id); - assert_ne!( - classify(p.dag(), &p.query_roots).0, - Q2Option::Exact, - "{invalid:?}" - ); - } - let cms = &physical_with(&run, Q2Option::CountMinHeapPerJob).id; - assert!( - invalid[cms.as_str()].contains("no guarantee"), - "{invalid:?}" - ); - let count_sketch = &physical_with(&run, Q2Option::CountSketchHeapPerJob).id; - assert!( - invalid[count_sketch.as_str()].contains("no guarantee"), - "{invalid:?}" - ); -} - -/// Without the counter declaration, Count-Min's weights are not proven -/// non-negative: plan selection reads no type from the `_total` name. -#[test] -fn stage3_plan_selection_count_min_needs_the_counter_declaration() { - let mut workload = example1(); - workload - .data_workload - .as_mut() - .unwrap() - .sources - .get_mut(EXAMPLE1_METRIC) - .unwrap() - .metric_type = None; - let run = rule_fixture(&workload); - let invalid = run.invalid(); - let cms = &physical_with(&run, Q2Option::CountMinHeapPerJob).id; - assert!( - invalid[cms.as_str()].contains("non-negative"), - "{invalid:?}" - ); -} - -/// A shared NotMaterialized input is charged once per consumer, and both -/// queries read the same 1 m window, so the shared-input candidates cost -/// what the independent one does; their totals differ from it only by -/// floating-point rounding (the same charges summed in another grouping). -/// Built once per evaluation (`MatAtQuery`), the shared input is read once: -/// the shared time range (the doc's "Raw with a shared input") is selected, -/// at 4.38 instead of 7.01 per second. -#[test] -fn stage3_plan_selection_shared_input_ties_up_to_rounding_unless_built_once() { - let run = pipeline(); - let independent = physical_with(&run, Q2Option::Exact); - let base = run.cost(&independent.id).unwrap(); - let exact_shared = |p: &&Physical| { - whole_windows(p.dag()) && classify(p.dag(), &p.query_roots) == (Q2Option::Exact, true) - }; - let built_once = |p: &Physical| { - p.dag() - .nodes - .iter() - .any(|n| materialization(p, n.id) == MaterializationOption::MatAtQuery) - }; - let shared: Vec<_> = run - .physical - .iter() - .filter(exact_shared) - .filter(|p| !built_once(p)) - .map(|p| (p.id.clone(), run.cost(&p.id).unwrap())) - .collect(); - assert!(!shared.is_empty()); - assert!( - shared - .iter() - .any(|(_, total)| total.to_bits() != base.to_bits()), - "the rounding difference the tie tolerance absorbs: {base} vs {shared:?}" - ); - for (id, total) in &shared { - assert!( - (total - base).abs() <= 1e-9 * base, - "{id}: {total} vs {base}" - ); - } - let selected = run.physical(&run.selection.selected); - assert!(exact_shared(&selected) && built_once(selected)); - let time_range = selected - .dag() - .nodes - .iter() - .find(|n| materialization(selected, n.id) == MaterializationOption::MatAtQuery) - .unwrap(); - assert_eq!( - relational(&time_range.payload).as_deref(), - Some("time_range") - ); - let cost = run.cost(&selected.id).unwrap(); - assert!( - (cost - 4.3770).abs() < 1e-3 && (base - 7.0125).abs() < 1e-3, - "{cost} vs {base}" - ); -} - -/// Tumbling windows rebuilt at every evaluation cost more than the whole -/// window: the tumbling windows read the same rows and update the same states, and -/// merging them adds work. Heap tumbling windows are invalid like their whole-window -/// builds (and their merged top-k is only approximate). -#[test] -fn stage3_plan_selection_tumbling_windows_at_query_time_cost_more() { - let run = pipeline(); - let selected = run.cost(&run.selection.selected).unwrap(); - let invalid = run.invalid(); - let mut compared = 0; - for p in run.physical.iter().filter(|p| !whole_windows(p.dag())) { - match run.cost(&p.id) { - Some(cost) => { - assert!(cost > selected * (1.0 + 1e-9), "{}: {cost}", p.id); - compared += 1; - } - None => assert_ne!( - classify(p.dag(), &p.query_roots).0, - Q2Option::Exact, - "{}: {invalid:?}", - p.id - ), - } - } - assert!(compared > 0); -} - -/// The selected plan is the cheapest valid candidate for the whole workload. -#[test] -fn stage3_plan_selection_selects_cheapest_valid() { - assert_selects_cheapest_valid(&pipeline()); -} - -/// Every node of a valid candidate is charged once per consumer that reads -/// it: a shared `NotMaterialized` node is recomputed for each query, and a -/// materialized one is computed once. -#[test] -fn stage3_plan_selection_charges_each_node_per_consuming_query() { - assert_each_node_charged_per_consuming_query(&pipeline()); -} - -/// A hand-built candidate with a materialized node: Q1's plan read by two -/// queries, its `time_range` kept `MatAtQuery`. Plan selection charges the -/// two accumulators (rate, sum) and their finalizations twice, once per query; the walks from -/// both roots stop at the materialized `time_range`, which is computed once -/// and reads the scan once. -fn materialized_case() -> Run { - let mut workload = example1(); - let repeating = workload.query_workload.repeating_queries.as_mut().unwrap(); - repeating.truncate(1); - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let roots = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let exact = compose_logical_candidate(&local, &vec![0; local.targets.len()]).unwrap(); - let run = run_explicit_candidates(&workload, roots, vec![exact], PlanningModels::builtin()); - let mut p = run.physical[0].clone(); - p.candidate.roots = vec![p.candidate.roots[0].clone(); 2]; - let dag = &mut p.candidate.dag; - dag.roots = vec![dag.roots[0]; 2]; - p.query_roots = dag.roots.clone(); - let time_range = dag - .nodes - .iter_mut() - .find(|n| relational(&n.payload).as_deref() == Some("time_range")) - .unwrap(); - time_range.materialization = Materialization::MatAtQuery(Medium::Memory); - let selection = select_plan( - std::slice::from_ref(&p.candidate), - &[exact_every_10s(), exact_every_10s()], - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin(), - ) - .unwrap(); - Run { - physical: vec![p], - selection, - ..run - } -} - -/// The charging helper agrees with plan selection when a node is -/// materialized. -#[test] -fn stage3_plan_selection_charging_helper_follows_plan_selection_with_a_materialized_node() { - let run = materialized_case(); - let p = &run.physical[0]; - let mut by_kind: Vec<_> = run.selection.costs[&p.id] - .per_node - .iter() - .map(|(id, c)| (operation(p.dag().payload(*id)), c.multiplier)) - .collect(); - by_kind.sort(); - let expected = [ - ("finalize_exact_accumulator", 2), - ("finalize_exact_accumulator", 2), - ("scan", 1), - ("summary_agg", 2), - ("summary_agg", 2), - ("time_range", 1), - ]; - assert_eq!( - by_kind, - expected.map(|(kind, times)| (kind.to_string(), times)) - ); - assert_each_node_charged_per_consuming_query(&run); -} - -/// Charging the scan under a materialized node once per query is caught. -#[test] -#[should_panic(expected = "multiplier of")] -fn stage3_plan_selection_charging_helper_rejects_recounting_below_a_materialized_node() { - let mut run = materialized_case(); - let p = run.physical[0].clone(); - let scan = p - .dag() - .nodes - .iter() - .find(|n| relational(&n.payload).as_deref() == Some("scan")) - .unwrap() - .id; - run.selection - .costs - .get_mut(&p.id) - .unwrap() - .per_node - .get_mut(&scan) - .unwrap() - .multiplier = 2; - assert_each_node_charged_per_consuming_query(&run); -} - -/// The plan selection picks compiles in the physical planner (added by the -/// implementer, not part of the spec). -#[test] -fn stage3_plan_selection_selected_plan_compiles_in_the_physical_planner() { - let run = pipeline(); - compile_in_runtime(run.physical(&run.selection.selected)).unwrap(); -} - -/// Sharing the input never costs more than reading it separately. -#[test] -fn stage3_plan_selection_shared_input_is_not_costlier() { - let run = pipeline(); - let by_combo: BTreeMap<_, _> = run - .physical - .iter() - .filter(|p| whole_windows(p.dag())) - .filter_map(|p| Some((classify(p.dag(), &p.query_roots), run.cost(&p.id)?))) - .collect(); - let mut compared = 0; - for option in PASS1_Q2_OPTIONS { - if let (Some(shared), Some(separate)) = ( - by_combo.get(&(option, true)), - by_combo.get(&(option, false)), - ) { - assert!(shared <= separate, "{option:?}"); - compared += 1; - } - } - assert!(compared > 0); -} diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs deleted file mode 100644 index 9c09da9fd..000000000 --- a/crates/integration-tests/tests/planner_layering_example2.rs +++ /dev/null @@ -1,1164 +0,0 @@ -//! Rule-fixture and production acceptance for Example 2: shared summaries across statistics. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::physical_common; -use crate::planner_layering_common; - -use std::collections::{BTreeMap, BTreeSet}; - -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use asap_types::ir::properties::ProbabilityExpr; -use asap_types::ir::properties::{AccuracyScope, AccuracyTarget}; -use asap_types::ir::schema::DataType; -use asap_types::ir::schema::{Field, Schema, SketchAlgorithm, SketchParams, SketchStatistic}; -use asap_types::ir::QueryRoot; -use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, - SourceFacts, SqlDialect, -}; -use planner_layering_common::*; - -// ── Example 2 workload ─────────────────────────────────────────────────── - -const SQL_Q1: &str = - "SELECT COUNT(DISTINCT src_ip) FROM flows WHERE ts >= now() - INTERVAL '1 minute'"; -const SQL_Q2: &str = "SELECT -SUM(p * LN(p)) FROM (\ - SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows \ - WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; -/// Q3 casts the counts to DOUBLE: the design's Int64 `SUM(c * c)` can -/// overflow, so the SQL frontend recognizes only the Float64 product. -const SQL_Q3: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (\ - SELECT src_ip, COUNT(*) AS c FROM flows \ - WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; - -/// The metric the PromQL stand-in reads. -const STANDIN_METRIC: &str = "flows_src_ip"; -/// PromQL stand-in for Q1–Q3 (see the module docs). -const PROMQL_Q1: &str = "distinct_over_time(flows_src_ip[1m])"; -const PROMQL_Q2: &str = "entropy_over_time(flows_src_ip[1m])"; -const PROMQL_Q3: &str = "l2_over_time(flows_src_ip[1m])"; - -/// (ε, δ) of Q1, Q2, Q3. Q3 is the strictest. -const ACCURACY: [(f64, f64); 3] = [(0.02, 0.01), (0.05, 0.01), (0.01, 0.01)]; - -fn panel(query: &str, (epsilon, delta): (f64, f64), scope: AccuracyScope) -> RepeatingEntry { - RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget { - accuracy_scope: scope, - ..AccuracyTarget::max_error(epsilon).failure_probability(delta) - }, - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - } -} - -/// The shared data workload with 10,000,000 distinct source IPs, in the -/// SQL `flows` table and the PromQL stand-in's `flows_src_ip` metric -/// (sampled every 15 s; SQL needs no sample interval). -fn data_workload() -> DataWorkload { - let ten_million = SourceFacts { - series_count: Some(10_000_000), - ..SourceFacts::default() - }; - let sampled = SourceFacts { - sample_interval: Some(DurationMs(15_000)), - ..ten_million.clone() - }; - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000_000.0 / 15.0, - }, - sources: [ - ("flows".to_string(), ten_million), - (STANDIN_METRIC.to_string(), sampled), - ] - .into(), - } -} - -/// The SQL queries' targets cover their whole result, as the doc's do. The -/// PromQL stand-in states them per value: it reads one series, but the -/// series count of `flows_src_ip` (the 10M source IPs) is what plan -/// selection takes as the number of values a PromQL vector returns, so a -/// whole-result target would be checked over 10M values (#669b). -fn workload(language: QueryLanguage, queries: [&str; 3]) -> PlanningWorkload { - let sql = matches!(language, QueryLanguage::SQL(_)); - let scope = if sql { - AccuracyScope::WholeResult - } else { - AccuracyScope::PerValue - }; - PlanningWorkload { - query_workload: QueryWorkload { - language, - query_batch: None, - repeating_queries: Some( - queries - .iter() - .zip(ACCURACY) - .map(|(q, accuracy)| panel(q, accuracy, scope)) - .collect(), - ), - }, - data_workload: Some(data_workload()), - } -} - -/// Example 2 as the doc writes it. -fn sql_workload() -> PlanningWorkload { - workload( - QueryLanguage::SQL(SqlDialect::DataFusionSQL), - [SQL_Q1, SQL_Q2, SQL_Q3], - ) -} - -fn standin_workload() -> PlanningWorkload { - workload(QueryLanguage::PromQL, [PROMQL_Q1, PROMQL_Q2, PROMQL_Q3]) -} - -/// `flows(ts, src_ip)`: the doc names only these two columns. -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "flows", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("src_ip", DataType::Utf8, false), - ], - 0, - vec![], - ), - ) -} - -async fn lower_sql(workload: &PlanningWorkload) -> Vec { - let mut roots = Vec::new(); - for entry in workload.query_workload.entries() { - let root = lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap_or_else(|e| panic!("{}: {e}", entry.query.0)); - roots.push(QueryRoot::Operator(root)); - } - roots -} - -// Targeted rule fixtures: exact, each local summary in isolation, and every -// query on its own UnivMon, needed by compatibility and cost assertions. -// Shared UnivMons come from the memo (`shared_pipeline`). -// Production optimization is tested separately; this is not a Cartesian search. -fn rule_fixture(workload: &PlanningWorkload, roots: Vec) -> Run { - let computations = rule_fixture_computations(workload, &roots); - run_explicit_candidates( - workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -/// [`rule_fixture`]'s plans, before Stage 2 and plan selection. -fn rule_fixture_computations( - workload: &PlanningWorkload, - roots: &[QueryRoot], -) -> Vec> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(roots), - ) - .unwrap(); - let exact = vec![0; local.targets.len()]; - let mut choices = BTreeSet::from([exact.clone()]); - let mut univmon = exact.clone(); - let mut shared_targets = Vec::new(); - for (target, options) in local.targets.iter().enumerate() { - for option in 1..options.choice_count() { - let mut choice = exact.clone(); - choice[target] = option; - choices.insert(choice); - } - if let Some(option) = (0..options.choice_count()).find(|&choice| { - matches!( - options.computation_for(choice), - Some(( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::UnivMon), - _ - )) - ) - }) { - univmon[target] = option; - shared_targets.push(target); - } - } - choices.insert(univmon.clone()); - for &third in &shared_targets { - for option in 0..local.targets[third].choice_count() { - let mut choice = univmon.clone(); - choice[third] = option; - choices.insert(choice); - } - } - let grid: Vec<_> = workload - .query_workload - .entries() - .map(|entry| entry.evaluation_grid_ms()) - .collect(); - let mut computations = Vec::new(); - for choice in choices { - if local.absorbed_choice_varies(&choice) { - continue; - } - let whole: Vec<_> = compose_logical_candidate(&local, &choice).unwrap(); - computations.push(whole.clone()); - let shared = share_identical(&whole); - if distinct_nodes(&shared) < distinct_nodes(&whole) { - computations.push(shared); - } - let tumbling_windows = tumbling_window_candidates(&whole, &grid); - // Isolate one tumbling-window configuration at a time; no Cartesian products of - // independent window implementations are needed for the error tests. - for tumbling_window in tumbling_windows { - computations.extend( - window_composition_pass2_candidates(&whole, &[tumbling_window]) - .unwrap() - .into_iter() - .map(|candidate| candidate.roots.unwrap()), - ); - } - } - computations -} - -#[test] -fn production_memo_selects_a_validated_example2_plan() { - let workload = standin_workload(); - let run = run_stages(&workload, lower_promql(&workload)); - assert_selects_cheapest_valid(&run); - assert_eq!(run.selection.accepted[&run.selection.selected].len(), 3); -} - -fn pipeline() -> Run { - let workload = standin_workload(); - rule_fixture(&workload, lower_promql(&workload)) -} - -/// The stand-in with the deployment's bound on distinct items per -/// evaluation of its one source, when given. -fn standin_with_distinct_bound(bound: Option) -> PlanningWorkload { - let mut workload = standin_workload(); - if let Some(bound) = bound { - workload - .data_workload - .as_mut() - .expect("data workload") - .sources - .get_mut(STANDIN_METRIC) - .expect("the stand-in's source") - .max_distinct_per_evaluation = Some(bound); - } - workload -} - -fn pipeline_with_distinct_bound(bound: Option) -> Run { - let workload = standin_with_distinct_bound(bound); - rule_fixture(&workload, lower_promql(&workload)) -} - -/// Plans composed of the memo's candidates, priced by Stage 2 and plan -/// selection: every query on its group's first candidate; each query alone -/// on its own UnivMon; and, per UnivMon state group the memo shares between -/// queries, those queries reading it. -fn memo_pipeline() -> Run { - let workload = standin_workload(); - let roots = lower_promql(&workload); - let computations = memo_computations(&workload, &roots); - run_explicit_candidates( - &workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -/// [`rule_fixture`]'s plans and [`memo_pipeline`]'s together, so the -/// memo's shared UnivMons compete with every query's own options. -fn shared_pipeline(workload: &PlanningWorkload, roots: Vec) -> Run { - let mut computations = rule_fixture_computations(workload, &roots); - computations.extend(memo_computations(workload, &roots)); - run_explicit_candidates( - workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -/// [`shared_pipeline`] over the stand-in. -fn standin_shared_pipeline() -> Run { - let workload = standin_workload(); - shared_pipeline(&workload, lower_promql(&workload)) -} - -/// [`memo_pipeline`]'s plans of `roots`, before Stage 2 and plan selection. -fn memo_computations(workload: &PlanningWorkload, roots: &[QueryRoot]) -> Vec> { - use asap_integration_tests::planning::{compose_memo_plan, expanded_memo, own_choices}; - use asap_types::ir::schema::FieldDataType; - use asap_types::ir::{ASAPOp, OperatorNode}; - use asap_types::stages::{CandidateForMemoGroup, OriginOfCandidateForMemoGroup}; - let entries: Vec<_> = workload.query_workload.entries().collect(); - let memo = expanded_memo(roots.to_vec(), &entries); - let queries = roots.len(); - let is_univmon = |node: &std::rc::Rc| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if *kind.algorithm() == SketchAlgorithm::UnivMon) - }; - // Query `q`'s topmost own group's first candidate `keep` accepts. - let own = |q: usize, keep: &dyn Fn(&CandidateForMemoGroup) -> bool| { - own_choices(&memo, q, keep).into_iter().next() - }; - let mut choices = vec![BTreeMap::new()]; - for q in 0..queries { - if let Some(choice) = own(q, &|c| { - c.origin == OriginOfCandidateForMemoGroup::Pass1 - && OperatorNode::reachable(&c.root_node).iter().any(is_univmon) - }) { - choices.push(BTreeMap::from([choice])); - } - } - for (state, group) in memo.groups.iter().enumerate() { - if !is_univmon(&group.logical_node) || group.reached_by_queries().len() < 2 { - continue; - } - let reading: Option> = group - .reached_by_queries() - .iter() - .map(|&q| { - own(q, &|c| { - c.input_groups.iter().any(|child| child.0 as usize == state) - }) - }) - .collect(); - choices.extend(reading); - } - choices - .iter() - .map(|choice| { - compose_memo_plan(&memo, queries, choice) - .into_iter() - .map(QueryRoot::Operator) - .collect() - }) - .collect() -} - -// ── Helpers ────────────────────────────────────────────────────────────── - -/// UnivMon build nodes of `c` with the queries that read each. -fn univmons(c: &Logical) -> Vec<(BTreeSet, SketchParams, BTreeSet)> { - sketch_builds(&c.dag) - .into_iter() - .filter(|(_, algorithm, _)| *algorithm == SketchAlgorithm::UnivMon) - .map(|(id, _, params)| { - let statistics = estimates_of(&c.dag, id) - .into_iter() - .map(|(_, s)| format!("{s:?}")) - .collect(); - (readers(&c.dag, &c.query_roots, id), params, statistics) - }) - .collect() -} - -/// The UnivMon read by every query in `queries` and by no other. -fn shared_univmon(c: &Logical, queries: &[usize]) -> Option { - let want: BTreeSet<_> = queries.iter().copied().collect(); - univmons(c) - .into_iter() - .find(|(readers, ..)| *readers == want) - .map(|(_, params, _)| params) -} - -/// Whether some summary build is read by more than one query. -fn shares_a_summary(c: &Logical) -> bool { - sketch_builds(&c.dag) - .iter() - .any(|(id, ..)| readers(&c.dag, &c.query_roots, *id).len() > 1) -} - -fn options(run: &Run, query: usize) -> BTreeSet { - run.logical - .iter() - .filter(|c| !shares_a_summary(c)) - .map(|c| query_option(&c.dag, &c.query_roots, query)) - .collect() -} - -/// UnivMon state size: heap entries plus counters over every layer. -fn footprint(params: &SketchParams) -> u64 { - let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - } = params - else { - panic!("UnivMon params: {params:?}"); - }; - u64::from(*heap_size) + u64::from(*sketch_rows) * u64::from(*sketch_cols) * u64::from(*layers) -} - -/// Q`query`'s own UnivMon parameters, from an independent candidate. -fn own_univmon(run: &Run, query: usize) -> SketchParams { - run.logical - .iter() - .filter(|c| !shares_a_summary(c)) - .find_map(|c| shared_univmon(c, &[query])) - .expect("Pass 1 offers UnivMon") -} - -// ── Workload and query-language frontends ───────────────────────────────────────────────── - -/// The encoded SQL workload is valid and normalizes to Q1, Q2, Q3. -#[test] -fn workload_encodes_example2() { - let workload = sql_workload(); - workload.validate().expect("valid workload"); - let queries: Vec<_> = workload - .query_workload - .entries() - .map(|e| e.query.0) - .collect(); - assert_eq!(queries, [SQL_Q1, SQL_Q2, SQL_Q3]); - standin_workload().validate().expect("valid stand-in"); -} - -/// The SQL frontend lowers Q1 to a distinct count over `flows`. -#[tokio::test] -async fn stage0_query_language_frontends_sql_q1_lowers_to_a_distinct_count() { - let roots = lower_sql(&sql_workload()).await; - let ops = frontend_operations(&roots[0]); - assert!(ops.contains(&"scan".to_string()), "{ops:?}"); - assert!( - ops.contains(&"aggregate:cardinality".to_string()), - "{ops:?}" - ); -} - -/// The SQL frontend recognizes Q2 as `Entropy(src_ip)` and Q3 as `L2(src_ip)`. -#[tokio::test] -async fn stage0_query_language_frontends_sql_q2_q3_lower_to_entropy_and_l2() { - let roots = lower_sql(&sql_workload()).await; - assert!(frontend_operations(&roots[1]).contains(&"aggregate:frequency_entropy".to_string())); - assert!(frontend_operations(&roots[2]).contains(&"aggregate:frequency_l2".to_string())); -} - -/// Once the SQL forms are recognized, the SQL workload has the stand-in's Pass 1 options. -#[tokio::test] -async fn stage1_logical_asap_optimization_sql_has_the_standin_options() { - let workload = sql_workload(); - let sql = rule_fixture(&workload, lower_sql(&workload).await); - let standin = pipeline(); - for q in 0..3 { - assert_eq!(options(&sql, q), options(&standin, q), "Q{}", q + 1); - } -} - -/// The doc's SQL workload itself gets the memo's shared candidate: one -/// UnivMon read by Q1, Q2 and Q3. -#[tokio::test] -async fn stage1_logical_asap_optimization_sql_adds_one_univmon_for_all_three() { - let workload = sql_workload(); - let run = shared_pipeline(&workload, lower_sql(&workload).await); - assert!( - run.logical.iter().any(|c| { - let ums = univmons(c); - ums.len() == 1 && ums[0].0 == BTreeSet::from([0, 1, 2]) - }), - "no SQL candidate reads one UnivMon for Q1, Q2 and Q3" - ); -} - -/// The stand-in lowers to the three statistics, each over a 1-min range of one input. -#[test] -fn stage0_query_language_frontends_standin_lowers_to_three_statistics_of_one_input() { - let roots = lower_promql(&standin_workload()); - let ops: Vec<_> = roots.iter().map(frontend_operations).collect(); - for (ops, statistic) in ops.iter().zip([ - "aggregate:cardinality", - "aggregate:frequency_entropy", - "aggregate:frequency_l2", - ]) { - let mut want = vec!["scan".to_string(), statistic.into(), "time_range".into()]; - want.sort(); - assert_eq!(ops, &want); - } -} - -/// The frontends give one summary-free workload DAG whose queries share no node. -#[test] -fn stage0_query_language_frontends_one_summary_free_dag_without_sharing() { - let run = pipeline(); - let stage0_query_language_frontends = &run.stage0_query_language_frontends; - assert_eq!(stage0_query_language_frontends.query_roots.len(), 3); - assert!(stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); - assert!(cross_query_nodes( - &stage0_query_language_frontends.dag, - &stage0_query_language_frontends.query_roots - ) - .is_empty()); -} - -// ── Logical ASAP-aware optimization ────────────────────────────────────── - -/// Pass 1 offers an exact and a UnivMon option for each of the three statistics. -#[test] -fn stage1_logical_asap_optimization_pass1_offers_exact_and_univmon_for_each_statistic() { - let run = pipeline(); - for q in 0..3 { - let found = options(&run, q); - assert!(found.contains("exact"), "Q{}: {found:?}", q + 1); - assert!(found.contains("UnivMon"), "Q{}: {found:?}", q + 1); - } -} - -/// Pass 1 offers a specialized distinct-count summary for Q1. -#[test] -fn stage1_logical_asap_optimization_pass1_offers_a_distinct_count_summary() { - let found = options(&pipeline(), 0); - assert!( - found.iter().any(|o| ["Hll", "Kmv"].contains(&o.as_str())), - "{found:?}" - ); -} - -/// Pass 1 offers a specialized entropy summary for Q2 and a norm summary for Q3. -#[test] -#[ignore = "needs Pass 1 specialized entropy and L2 summary families (#444)"] -fn stage1_logical_asap_optimization_pass1_offers_specialized_entropy_and_l2_summaries() { - let run = pipeline(); - for q in [1, 2] { - let found = options(&run, q); - assert!( - found.iter().any(|o| o != "exact" && o != "UnivMon"), - "Q{}: {found:?}", - q + 1 - ); - } -} - -/// The summary-capability rule adds one UnivMon build feeding a distinct-count, an entropy and an L2 estimate. -#[test] -fn stage1_logical_asap_optimization_summary_capability_adds_one_univmon_for_all_three() { - let run = memo_pipeline(); - let shared = run.logical.iter().find(|c| { - let ums = univmons(c); - ums.len() == 1 && ums[0].0 == BTreeSet::from([0, 1, 2]) - }); - let c = shared.expect("a candidate with one UnivMon read by Q1, Q2 and Q3"); - let statistics = &univmons(c)[0].2; - for s in [ - SketchStatistic::Cardinality, - SketchStatistic::FrequencyEntropy, - SketchStatistic::FrequencyL2, - ] { - assert!(statistics.contains(&format!("{s:?}")), "{statistics:?}"); - } -} - -/// A UnivMon shared by several statistics is sized for every consumer: -/// each consumer is sized alone, and the shared build takes each -/// parameter's largest value among them. -#[test] -fn stage1_logical_asap_optimization_shared_univmon_takes_each_parameters_largest_own_size() { - let run = memo_pipeline(); - let own = |q| match own_univmon(&run, q) { - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - } => [heap_size, sketch_rows, sketch_cols, u32::from(layers)], - other => panic!("{other:?}"), - }; - let mut seen = 0; - for queries in [vec![0, 1, 2], vec![0, 1], vec![0, 2], vec![1, 2]] { - let mut want = [0u32; 4]; - for &q in &queries { - for (w, p) in want.iter_mut().zip(own(q)) { - *w = (*w).max(p); - } - } - let want = SketchParams::UnivMon { - heap_size: want[0], - sketch_rows: want[1], - sketch_cols: want[2], - layers: want[3] as u8, - // Ignored by equality. - distinct_failure_probability: None, - }; - for c in &run.logical { - if let Some(params) = shared_univmon(c, &queries) { - seen += 1; - assert_eq!(params, want, "{} {queries:?}", c.id); - } - } - } - assert!(seen > 0, "no shared UnivMon"); -} - -/// Pass 1 sizes UnivMon for its query's target: ε = 0.01 (Q3) needs a larger state than ε = 0.05 (Q2). -#[test] -fn stage1_logical_asap_optimization_univmon_is_sized_per_accuracy_target() { - let run = pipeline(); - assert!(footprint(&own_univmon(&run, 2)) > footprint(&own_univmon(&run, 1))); -} - -/// Only a UnivMon, never a specialized summary, is shared across the statistics. -#[test] -fn stage1_logical_asap_optimization_only_univmon_is_shared_across_statistics() { - let run = standin_shared_pipeline(); - for c in &run.logical { - for (id, algorithm, _) in sketch_builds(&c.dag) { - if readers(&c.dag, &c.query_roots, id).len() > 1 { - assert_eq!(algorithm, SketchAlgorithm::UnivMon, "{}", c.id); - } - } - } -} - -/// Logical candidates are valid DAGs with unique ids. -#[test] -fn stage1_logical_asap_optimization_candidates_are_valid_and_uniquely_named() { - assert_valid_and_uniquely_named(&pipeline()); -} - -// ── Physical ASAP-aware optimization ───────────────────────────────────── - -/// No candidate is discarded before plan selection: each logical candidate keeps one physical candidate with nothing materialized, plus one per other materialization assignment. -#[test] -fn stage2_physical_asap_optimization_keeps_every_logical_candidate() { - assert_one_unmaterialized_physical_candidate_per_logical(&pipeline()); -} - -/// Physical ASAP-aware optimization keeps the shared UnivMon as one build node read by all three queries. -#[test] -fn stage2_physical_asap_optimization_keeps_the_shared_univmon_as_one_build() { - let run = memo_pipeline(); - let mut seen = 0; - for c in run - .logical - .iter() - .filter(|c| shared_univmon(c, &[0, 1, 2]).is_some()) - { - for p in run.physical_of(c) { - seen += 1; - let builds: Vec<_> = sketch_builds(p.dag()) - .into_iter() - .filter(|(_, a, _)| *a == SketchAlgorithm::UnivMon) - .collect(); - assert_eq!(builds.len(), 1, "{}", p.id); - assert_eq!(readers(p.dag(), &p.query_roots, builds[0].0).len(), 3); - } - } - assert!(seen > 0); -} - -// ── Plan selection ─────────────────────────────────────────────────────── - -/// Plan selection selects one candidate and gives every other one a reason. -#[test] -fn stage3_plan_selection_selects_one_and_explains_the_rest() { - assert_selects_one_and_explains_the_rest(&pipeline()); -} - -/// The selected plan is the cheapest valid candidate for the whole workload. -#[test] -fn stage3_plan_selection_selects_cheapest_valid() { - assert_selects_cheapest_valid(&pipeline()); -} - -/// Every node of a valid candidate is charged once per query that reads it: nothing is materialized, so a shared node is recomputed for each. -#[test] -fn stage3_plan_selection_charges_each_node_per_consuming_query() { - assert_each_node_charged_per_consuming_query(&pipeline()); -} - -/// `asap_sketchlib` has no Theta sketch, so no candidate builds one. -#[test] -fn stage3_plan_selection_has_no_theta_candidate() { - let run = pipeline(); - for (id, cost) in &run.selection.costs { - assert!( - !cost - .per_node - .values() - .any(|n| n.detail.starts_with("build Theta")), - "{id} builds Theta" - ); - } -} - -/// A UnivMon estimate has a guarantee, so plan selection judges it against its target instead of rejecting it for having none. -#[test] -#[ignore = "needs a UnivMon entropy accuracy model (#788); L2 and the distinct count have one"] -fn stage3_plan_selection_judges_univmon_with_an_accuracy_model() { - let run = pipeline(); - let invalid = run.invalid(); - for c in &run.logical { - for p in run.physical_of(c) { - let Some(reason) = invalid.get(p.id.as_str()) else { - continue; - }; - for q in 0..3 { - if query_option(&c.dag, &c.query_roots, q) == "UnivMon" { - let unguaranteed = format!("q{}: the output has no guarantee", q + 1); - assert!(!reason.starts_with(&unguaranteed), "{}: {reason}", p.id); - } - } - } - } -} - -/// The physical candidates that answer Q1 with `algorithm` and Q2 and Q3 -/// exactly: their validity depends only on Q1's estimate. -fn q1_only_candidates<'a>(run: &'a Run, algorithm: &str) -> Vec<&'a str> { - let ids: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3) - .map(|q| query_option(&c.dag, &c.query_roots, q)) - .eq([algorithm, "exact", "exact"]) - }) - .flat_map(|c| run.physical_of(c).map(|p| p.id.as_str())) - .collect(); - assert!( - !ids.is_empty(), - "a candidate answers Q1 with {algorithm} alone" - ); - ids -} - -fn hll_only_candidates(run: &Run) -> Vec<&str> { - q1_only_candidates(run, "Hll") -} - -/// With the deployment's bound on distinct items per evaluation, plan -/// selection binds classic HLL's failure probability and accepts Q1's HLL -/// estimate; without a bound it stays unknown and the target unmet. Pass 1 -/// sizes HLL from ε = 0.02 and δ = 0.01 (#694): `m ≥ 1/(2ε²δ)` gives -/// precision 17, which certifies δ = 0.01 at ε = 0.02 for bounds up to -/// 1,842 distinct items. -#[test] -fn stage3_plan_selection_accepts_hll_given_a_distinct_bound() { - for bound in [9, 1_842] { - let bounded = pipeline_with_distinct_bound(Some(bound)); - for id in hll_only_candidates(&bounded) { - assert!( - bounded.cost(id).is_some(), - "bound {bound}: {id}: {:?}", - bounded.invalid().get(id) - ); - } - } - let unbounded = pipeline_with_distinct_bound(None); - let invalid = unbounded.invalid(); - for id in hll_only_candidates(&unbounded) { - let reason = invalid.get(id).unwrap_or_else(|| panic!("{id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - assert!( - reason.contains("failure probability None"), - "{id}: {reason}" - ); - } -} - -/// A bound too large for the precision sized for ε alone (17 certifies at -/// most 1,842 distinct items) is met by a larger precision: Pass 1's sizing -/// for the failure probability raises HLL's precision until its failure -/// probability over the bound at ε = 0.02 meets the target, up to precision -/// 18 (#669b), which certifies up to 3,791; Stage 3 only checks it. Past -/// that, up to Example 2's 10M source IPs, classic HLL's linear-counting -/// contract certifies nothing. -#[test] -fn pass1_sizing_enlarges_hll_for_a_larger_distinct_bound() { - let run = pipeline_with_distinct_bound(Some(3_000)); - for id in hll_only_candidates(&run) { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } - for bound in [10_000, 10_000_000] { - let run = pipeline_with_distinct_bound(Some(bound)); - let invalid = run.invalid(); - for id in hll_only_candidates(&run) { - let reason = invalid - .get(id) - .unwrap_or_else(|| panic!("bound {bound}: {id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - assert!( - !reason.contains("failure probability None"), - "{id}: {reason}" - ); - } - } -} - -/// KMV certifies Q1 over the 10M source IPs without a distinct bound -/// (#694): its bound holds for any number of distinct items, and Pass 1 -/// sizes `k` for the requested δ (#700), so a stricter δ = 10⁻⁴ is met -/// too, by a larger `k`. -#[test] -fn stage3_plan_selection_accepts_kmv_for_any_failure_probability_without_a_bound() { - let mut ks = Vec::new(); - for delta in [0.01, 1e-4] { - let mut workload = standin_workload(); - let q1 = &mut workload - .query_workload - .repeating_queries - .as_mut() - .expect("repeating queries")[0]; - q1.requirements.accuracy = AccuracyTarget { - accuracy_scope: AccuracyScope::PerValue, - ..AccuracyTarget::max_error(0.02).failure_probability(delta) - }; - let run = rule_fixture(&workload, lower_promql(&workload)); - for id in q1_only_candidates(&run, "Kmv") { - assert!( - run.cost(id).is_some(), - "δ = {delta}: {id}: {:?}", - run.invalid().get(id) - ); - let k: BTreeSet<_> = sketch_builds(run.physical(id).dag()) - .into_iter() - .filter_map(|(_, _, params)| match params { - SketchParams::Kmv { k, .. } => Some(k), - _ => None, - }) - .collect(); - ks.push((delta, k)); - } - } - assert!( - ks.iter().any(|(d, k)| *d == 0.01 && k.contains(&26_656)), - "{ks:?}" - ); - assert!( - ks.iter() - .any(|(d, k)| *d == 1e-4 && k.iter().all(|k| *k > 26_656)), - "{ks:?}" - ); -} - -/// The physical candidates among `ids` that build their sketch as a -/// tumbling-window `SummaryTumblingWindowChain`. -fn tumbling<'a>(run: &Run, ids: Vec<&'a str>) -> Vec<&'a str> { - ids.into_iter() - .filter(|id| { - let dag = run.physical(id).dag(); - sketch_builds(dag).iter().any(|(build, _, _)| { - matches!(window_form(dag, *build), WindowForm::Tumbling { .. }) - }) - }) - .collect() -} - -/// The enlarged precision reaches a sketch carried by a tumbling-window -/// `SummaryTumblingWindowChain` too (#713): Q1's HLL over a bound of 3,000 is valid only at -/// precision 18, above the 17 sized for ε alone, so a valid tumbling-window -/// HLL candidate carries precision 18 in its chain. -#[test] -fn pass1_sizing_enlarges_tumbling_window_hll() { - let run = pipeline_with_distinct_bound(Some(3_000)); - let hll = tumbling(&run, hll_only_candidates(&run)); - assert!( - !hll.is_empty(), - "a tumbling-window HLL candidate answers Q1" - ); - for id in hll { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } -} - -/// Stage 1 does not read the distinct bound: its candidates are the same -/// with and without one, and Q1's HLL estimate leaves its failure -/// probability symbolic in the source's bound. -#[test] -fn stage1_logical_asap_optimization_leaves_the_hll_failure_probability_symbolic() { - let bounded = pipeline_with_distinct_bound(Some(9)); - let unbounded = pipeline_with_distinct_bound(None); - let stage1 = |run: &Run| { - run.logical - .iter() - .map(|c| format!("{} {:?}", c.id, c.roots)) - .collect::>() - }; - assert_eq!(stage1(&bounded), stage1(&unbounded)); - let mut seen = 0; - for c in unbounded - .logical - .iter() - .filter(|c| !shares_a_summary(c) && query_option(&c.dag, &c.query_roots, 0) == "Hll") - { - let QueryRoot::Operator(q1) = &c.roots[0] else { - panic!("operator root") - }; - let guarantee = q1.guarantee.as_ref().expect("Q1 has a guarantee"); - assert_eq!( - guarantee.spec.failure_probability, - ProbabilityExpr::Unknown { - statistic: - "classic_hll_failure_probability[p=17,eps=0.02](max_distinct[flows_src_ip])" - .into() - }, - "{}", - c.id - ); - seen += 1; - } - assert!(seen > 0, "a candidate answers Q1 with HLL"); -} - -/// The physical candidates that answer Q`query` with UnivMon and the other -/// two exactly: their validity depends only on that UnivMon estimate. -fn univmon_only_candidates(run: &Run, query: usize) -> Vec<&str> { - let ids: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3).all(|q| { - query_option(&c.dag, &c.query_roots, q) - == if q == query { "UnivMon" } else { "exact" } - }) - }) - .flat_map(|c| run.physical_of(c).map(|p| p.id.as_str())) - .collect(); - assert!( - !ids.is_empty(), - "a candidate answers Q{} with UnivMon alone", - query + 1 - ); - ids -} - -/// Q1's UnivMon is sized for (0.02, 0.01): a 337,500-key heap on 19 -/// layers. Its distinct count is certified once the deployment bounds the -/// source's distinct items, both at 9 (the top layer is always complete) -/// and at 10M (7 layers suffice, where classic HLL certifies nothing); -/// without a bound its failure probability stays unknown. -#[test] -fn stage3_plan_selection_accepts_univmon_distinct_given_a_distinct_bound() { - for bound in [9, 10_000_000] { - let run = pipeline_with_distinct_bound(Some(bound)); - for id in univmon_only_candidates(&run, 0) { - assert!( - run.cost(id).is_some(), - "{bound} {id}: {:?}", - run.invalid().get(id) - ); - } - } - let unbounded = pipeline_with_distinct_bound(None); - let invalid = unbounded.invalid(); - for id in univmon_only_candidates(&unbounded, 0) { - let reason = invalid.get(id).unwrap_or_else(|| panic!("{id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - assert!( - reason.contains("failure probability None"), - "{id}: {reason}" - ); - } -} - -/// A distinct bound past what 19 layers cover for Q1's heap -/// (337,500 · 2^17) certifies nothing. -#[test] -fn stage3_plan_selection_rejects_univmon_distinct_past_its_layers() { - let run = pipeline_with_distinct_bound(Some(337_500 << 17 | 1)); - let invalid = run.invalid(); - for id in univmon_only_candidates(&run, 0) { - let reason = invalid.get(id).unwrap_or_else(|| panic!("{id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - } -} - -/// Q3's L2 at (0.01, 0.01) is certified from layer 0 alone: a 5 × 56,817 -/// Count Sketch on one layer, with no data-workload statistic. -#[test] -fn stage3_plan_selection_accepts_univmon_l2() { - let run = pipeline(); - assert_eq!( - own_univmon(&run, 2), - SketchParams::UnivMon { - heap_size: 1, - sketch_rows: 5, - sketch_cols: 56_817, - layers: 1, - distinct_failure_probability: None, - } - ); - for id in univmon_only_candidates(&run, 2) { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } -} - -/// SQL Q3's L2 through UnivMon keeps its guarantee: the exact population -/// count joined beside it (a cross join, no key) and the `CASE` deciding the -/// empty-input NULL on that exact count keep the estimate's guarantee, so -/// plan selection accepts it. -#[tokio::test] -async fn stage3_plan_selection_accepts_sql_univmon_l2_beside_its_exact_population_count() { - let workload = sql_workload(); - let run = rule_fixture(&workload, lower_sql(&workload).await); - for id in univmon_only_candidates(&run, 2) { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } -} - -/// One materialized UnivMon for all three costs no more than three separate UnivMons. -#[test] -#[ignore = "needs a UnivMon entropy accuracy model (#788): without one, the shared UnivMon candidate is left unpriced"] -fn stage3_plan_selection_shared_univmon_costs_no_more_than_three() { - let run = standin_shared_pipeline(); - let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); - let separate: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3).all(|q| query_option(&c.dag, &c.query_roots, q) == "UnivMon") - }) - .map(|c| cost(c).unwrap_or_else(|| panic!("{} is not priced", c.id))) - .collect(); - assert!(!separate.is_empty(), "independent all-UnivMon candidate"); - let mut compared = 0; - for shared in run - .logical - .iter() - .filter(|c| shared_univmon(c, &[0, 1, 2]).is_some()) - { - let shared_cost = cost(shared).unwrap_or_else(|| panic!("{} is not priced", shared.id)); - for separate_cost in &separate { - assert!( - shared_cost <= *separate_cost, - "{shared_cost} vs {separate_cost}" - ); - } - compared += 1; - } - assert!(compared > 0); -} - -/// The selected plan updates at most one UnivMon per flow record: separate UnivMons are dominated by the shared one. -#[test] -fn stage3_plan_selection_selected_plan_has_at_most_one_univmon() { - let run = standin_shared_pipeline(); - let selected = run.physical(&run.selection.selected); - let count = sketch_builds(selected.dag()) - .iter() - .filter(|(_, a, _)| *a == SketchAlgorithm::UnivMon) - .count(); - assert!(count <= 1, "{}: {count} UnivMons", selected.id); -} - -/// Stage 3 over the candidates that answer Q1 with KMV and Q2 and Q3 -/// exactly selects one, and the executor runs its Q1: the distinct count of -/// the window's sample values. Below KMV's `k` distinct values its estimate -/// is exact. Only Q1 is compiled: the exact entropy and L2 of Q2 and Q3 do -/// not compile yet ("per-entity aggregate requires an explicit entity -/// binding"). -#[test] -fn stage3_plan_selection_selected_kmv_plan_executes_end_to_end() { - use asap_devtools::memo_pricing::reference_select as select_plan; - use asap_executor::physical_planner::{compile, promql_rows}; - use asap_executor::runtime::Scope; - use asap_executor::values::Value; - use asap_planner::PlanningModels; - - let workload = standin_workload(); - let roots = lower_promql(&workload); - let run = rule_fixture(&workload, roots.clone()); - let kmv: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3) - .map(|q| query_option(&c.dag, &c.query_roots, q)) - .eq(["Kmv", "exact", "exact"]) - }) - .flat_map(|c| run.physical_of(c).map(|p| p.candidate.clone())) - .collect(); - assert!(!kmv.is_empty(), "a candidate answers Q1 with KMV"); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let selection = select_plan( - &kmv, - &entries, - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin(), - ) - .expect("Stage 3 selects a KMV plan"); - let p = run.physical(&selection.selected); - assert_eq!(query_option(p.dag(), &p.query_roots, 0), "Kmv"); - - // 1,000 flow records over 50 s with 300 distinct source IPs. - let distinct = 300u32; - let samples: Vec<(i64, f64)> = (0..1_000u32) - .map(|i| (i64::from(i) * 50, f64::from(i * 7 % distinct))) - .collect(); - let labels = BTreeMap::from([("__name__".to_string(), "flows_src_ip".to_string())]); - // The whole plan runs at query time over the samples bound at its `Scan`. - let dag = p.dag(); - let store = physical_common::raw_data_store(dag, |_, schema| { - samples - .iter() - .map(|(ts, v)| promql_rows::series_row(schema, &labels, *ts, *v).unwrap()) - .collect() - }); - let q1 = p.query_roots[0] as u64; - let plan = compile(dag, &[], &[q1]).unwrap_or_else(|e| panic!("{}: {e}", p.id)); - let result = physical_common::execute( - &plan, - store, - Scope::Query { - evaluation_time_ms: 55_000, - }, - ); - let values: Vec = result[0] - .iter() - .flat_map(|batch| batch.to_rows()) - .flatten() - .filter_map(|value| match value { - Value::Float64(v) => Some(v), - _ => None, - }) - .collect(); - assert_eq!(values, [f64::from(distinct)], "{}", p.id); -} diff --git a/crates/integration-tests/tests/planner_layering_example3.rs b/crates/integration-tests/tests/planner_layering_example3.rs deleted file mode 100644 index c0c8ab6ed..000000000 --- a/crates/integration-tests/tests/planner_layering_example3.rs +++ /dev/null @@ -1,635 +0,0 @@ -//! Rule-fixture and production acceptance for Example 3: window implementations. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::planner_layering_common; - -use std::collections::BTreeSet; - -use asap_types::ir::flat::NodeId; -use asap_types::ir::schema::{SketchAlgorithm, SketchParams, SketchStatistic}; -use asap_types::workload::DataArrival; - -use planner_layering_common::*; - -// Fixed algorithm/window fixtures keep numerical evaluator checks independent -// of a time-bounded production search's archived incumbents. -fn run_a() -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ); - let roots = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut computations = Vec::new(); - for kll in [false, true] { - let choices: Vec<_> = local - .targets - .iter() - .map(|target| { - if kll { - (0..target.choice_count()) - .find(|&choice| { - matches!( - target.computation_for(choice), - Some(( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll), - _ - )) - ) - }) - .unwrap() - } else { - 0 - } - }) - .collect(); - let whole = unshare(&compose_logical_candidate(&local, &choices).unwrap()); - computations.push(whole.clone()); - computations.push(share_identical(&whole)); - } - run_explicit_candidates( - &workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -fn run_b() -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = pattern_b(); - let roots = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut computations = vec![roots.clone()]; - for option in 0..local.targets[0].choice_count() { - if !matches!( - local.targets[0].computation_for(option), - Some((HowToComputeAggregateMeasure::Summary(_), _)) - ) { - continue; - } - let whole: Vec<_> = compose_logical_candidate(&local, &[option]).unwrap(); - let tumbling_windows = tumbling_window_candidates(&whole, &[Some(PATTERN_B_INTERVAL_MS)]); - let chain = window_composition_pass2_candidates(&whole, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - computations.extend([whole, chain]); - } - run_explicit_candidates( - &workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -#[test] -fn production_memo_covers_pattern_b_with_validated_accuracy() { - let workload = pattern_b(); - let run = run_stages_under(&workload, lower_promql(&workload), &Default::default()); - assert_selects_cheapest_valid(&run); - assert_eq!(run.selection.accepted[&run.selection.selected].len(), 1); -} - -/// The per-computation options of `query` across all candidates. -fn options(run: &Run, query: usize) -> BTreeSet { - run.logical - .iter() - .map(|c| query_option(&c.dag, &c.query_roots, query)) - .collect() -} - -fn op(name: &str) -> String { - name.to_string() -} - -/// The summary builds of `c`: (build, algorithm, window form, readers). -fn windowed_builds(c: &Logical) -> Vec<(NodeId, SketchAlgorithm, WindowForm, BTreeSet)> { - sketch_builds(&c.dag) - .into_iter() - .map(|(id, algorithm, _)| { - let form = window_form(&c.dag, id); - (id, algorithm, form, readers(&c.dag, &c.query_roots, id)) - }) - .collect() -} - -// ── Workloads ──────────────────────────────────────────────────────────── - -/// Pattern A is one ad hoc batch of five queries run once at T; Pattern B one repeating query. -#[test] -fn workload_encodes_example3() { - let a = pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ); - a.validate().expect("valid Pattern A"); - let queries: Vec<_> = a.query_workload.entries().map(|e| e.query.0).collect(); - assert_eq!(queries, PATTERN_A.map(|pattern| pattern.promql)); - let b = pattern_b(); - b.validate().expect("valid Pattern B"); - assert_eq!(evaluation_interval_ms(&b, 0), Some(PATTERN_B_INTERVAL_MS)); -} - -// ── Pattern A: query-language frontends ─────────────────────────────── - -/// Each Pattern A query lowers to scan → (time shift) → range → quantile. -#[test] -fn stage0_query_language_frontends_a_lowers_each_query_to_its_interval() { - let roots = lower_promql(&pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - )); - let ops: Vec<_> = roots.iter().map(frontend_operations).collect(); - let plain = vec![op("aggregate:quantile"), op("scan"), op("time_range")]; - let shifted = vec![ - op("aggregate:quantile"), - op("scan"), - op("time_range"), - op("time_shift"), - ]; - assert_eq!( - ops, - [ - plain.clone(), - plain, - shifted.clone(), - shifted.clone(), - shifted - ] - ); -} - -/// The frontends give one summary-free DAG whose five queries share no node. -#[test] -fn stage0_query_language_frontends_a_one_summary_free_dag_without_sharing() { - let run = run_a(); - let stage0_query_language_frontends = &run.stage0_query_language_frontends; - assert_eq!(stage0_query_language_frontends.query_roots.len(), 5); - assert!(stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); - assert!(cross_query_nodes( - &stage0_query_language_frontends.dag, - &stage0_query_language_frontends.query_roots - ) - .is_empty()); -} - -// ── Pattern A: logical ASAP-aware optimization ─────────────────────────── - -/// Pass 1 offers each query an exact and a KLL candidate over its own interval. -#[test] -fn stage1_logical_asap_optimization_a_pass1_offers_exact_and_kll_per_query() { - let run = run_a(); - for q in 0..5 { - let found = options(&run, q); - assert!( - found.contains("exact") && found.contains("Kll"), - "q{}: {found:?}", - q + 1 - ); - } -} - -/// The five independent KLLs: one per query, each read by its query only, all sized for ε = 0.005. -#[test] -fn stage1_logical_asap_optimization_a_keeps_five_independent_klls() { - let run = run_a(); - let all_kll = run - .logical - .iter() - .find(|c| { - !c.shared_input() && (0..5).all(|q| query_option(&c.dag, &c.query_roots, q) == "Kll") - }) - .expect("independent all-KLL candidate"); - let builds = sketch_builds(&all_kll.dag); - assert_eq!(builds.len(), 5); - let readers: BTreeSet<_> = builds - .iter() - .map(|(id, ..)| readers(&all_kll.dag, &all_kll.query_roots, *id)) - .collect(); - assert_eq!(readers, (0..5).map(|q| BTreeSet::from([q])).collect()); - let sizes: BTreeSet<_> = builds - .iter() - .map(|(_, _, params)| match params { - SketchParams::Kll { k, .. } => *k, - other => panic!("{other:?}"), - }) - .collect(); - assert_eq!(sizes.len(), 1, "one ε, one KLL size: {sizes:?}"); -} - -/// The identical-expression rule shares only raw input (scan, range, shift), never a quantile or summary, and keeps the unshared variant. -#[test] -fn stage1_logical_asap_optimization_a_identical_expression_rule_shares_only_raw_input() { - let run = run_a(); - assert!(run.logical.iter().any(|c| c.shared_input())); - assert!(run.logical.iter().any(|c| !c.shared_input())); - for c in &run.logical { - for id in cross_query_nodes(&c.dag, &c.query_roots) { - let kind = relational(c.dag.payload(id)); - assert!( - matches!(kind.as_deref(), Some("scan" | "time_range" | "time_shift")), - "{}: shared {kind:?}", - c.id - ); - } - } -} - -/// One Exponential Histogram of KLLs over [T − 5y, T] serves all five queries, each through its own merge and p99 estimate. -#[test] -#[ignore = "needs Pass 2 window composition across queries: one window summary for all five (#628, #634)"] -fn stage1_logical_asap_optimization_a_window_composition_adds_one_eh_for_all_five() { - let run = run_a(); - let found = run.logical.iter().find(|c| { - windowed_builds(c).iter().any(|(_, a, form, readers)| { - *a == SketchAlgorithm::Kll - && matches!(form, WindowForm::ExponentialHistogram { horizon_ms } if *horizon_ms >= 5 * YEAR_MS) - && readers.len() == 5 - }) - }); - let c = found.expect("a candidate with one EH of KLLs read by all five queries"); - let (eh, ..) = windowed_builds(c) - .into_iter() - .find(|(_, _, form, _)| matches!(form, WindowForm::ExponentialHistogram { .. })) - .unwrap(); - let estimates = estimates_of(&c.dag, eh); - assert_eq!(estimates.len(), 5, "one estimate per query"); - for (estimate, statistic) in estimates { - assert_eq!(statistic, SketchStatistic::Quantile { q: 0.99 }); - let merged = c - .dag - .producers(estimate) - .into_iter() - .any(|p| is_merge(c.dag.payload(p))); - assert!( - merged, - "{}: estimate {estimate:?} does not read a merge", - c.id - ); - } -} - -/// The shared EH candidate is added next to the independent KLL candidates, not instead of them. -#[test] -#[ignore = "needs Pass 2 window composition across queries: one window summary for all five (#628, #634)"] -fn stage1_logical_asap_optimization_a_keeps_independent_and_shared_window_summaries() { - let run = run_a(); - let shared = run.logical.iter().any(|c| { - windowed_builds(c).iter().any(|(_, _, f, r)| { - matches!(f, WindowForm::ExponentialHistogram { .. }) && r.len() == 5 - }) - }); - let independent = run.logical.iter().any(|c| { - let builds = windowed_builds(c); - builds.len() == 5 - && builds.iter().all(|(_, a, f, r)| { - *a == SketchAlgorithm::Kll && *f == WindowForm::None && r.len() == 1 - }) - }); - assert!(shared && independent); -} - -/// Pass 2 adds a candidate for each way of grouping two queries onto one shared window summary. -#[test] -#[ignore = "needs Pass 2 window composition across queries: partial groupings (#628, #634)"] -fn stage1_logical_asap_optimization_a_window_composition_groups_every_pair() { - let run = run_a(); - let groups: BTreeSet<_> = run - .logical - .iter() - .flat_map(windowed_builds) - .filter(|(_, _, f, r)| *f != WindowForm::None && r.len() > 1) - .map(|(.., r)| r) - .collect(); - for a in 0..5 { - for b in a + 1..5 { - assert!( - groups.contains(&BTreeSet::from([a, b])), - "pair q{} q{}", - a + 1, - b + 1 - ); - } - } -} - -// ── Pattern A: plan selection ──────────────────────────────────────────── - -/// Plan selection selects one Pattern A candidate, explains the rest, and picks the cheapest valid one. -#[test] -fn stage3_plan_selection_a_selects_cheapest_valid() { - let run = run_a(); - assert_selects_one_and_explains_the_rest(&run); - assert_selects_cheapest_valid(&run); -} - -/// Every node of a valid candidate is charged once per query that reads it: nothing is materialized, so a shared node is recomputed for each. -#[test] -fn stage3_plan_selection_a_charges_each_node_per_consuming_query() { - assert_each_node_charged_per_consuming_query(&run_a()); -} - -/// Sharing the scan never costs more than scanning separately, for the same per-computation choices. -#[test] -#[ignore = "needs plan selection to charge a shared scan that is not materialized per query by the \ - window each query reads, not by the longest window it serves (#632)"] -fn stage3_plan_selection_a_shared_scan_is_not_costlier() { - let run = run_a(); - let key = |c: &Logical| { - (0..5) - .map(|q| query_option(&c.dag, &c.query_roots, q)) - .collect::>() - }; - let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); - let mut compared = 0; - for shared in run.logical.iter().filter(|c| c.shared_input()) { - let separate = run - .logical - .iter() - .find(|c| !c.shared_input() && key(c) == key(shared)) - .expect("independent counterpart"); - if let (Some(s), Some(i)) = (cost(shared), cost(separate)) { - assert!(s <= i, "{} {s} vs {} {i}", shared.id, separate.id); - compared += 1; - } - } - assert!(compared > 0); -} - -// ── Pattern B: query-language frontends and Pass 1 ──────────────────── - -/// Pattern B lowers to scan → range 5m → quantile, with no summary. -#[test] -fn stage0_query_language_frontends_b_lowers_to_one_range_quantile() { - let run = run_b(); - let roots = lower_promql(&pattern_b()); - assert_eq!( - frontend_operations(&roots[0]), - [op("aggregate:quantile"), op("scan"), op("time_range")] - ); - assert!(run - .stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); -} - -/// Pass 1 offers an exact and a KLL candidate for the 5-min window. -#[test] -fn stage1_logical_asap_optimization_b_pass1_offers_exact_and_kll() { - let found = options(&run_b(), 0); - assert!( - found.contains("exact") && found.contains("Kll"), - "{found:?}" - ); -} - -// ── Pattern B: window composition ──────────────────────────────────────── - -/// Each Pass 1 option gets three window forms: none, a 5-min sliding window with a 1-min slide, and 1-min tumbling windows. -#[test] -#[ignore = "needs Pass 2 window composition: sliding windows (#522); tumbling windows pass in \ - stage1_logical_asap_optimization_b_window_composition_adds_tumbling_per_mergeable_option"] -fn stage1_logical_asap_optimization_b_window_composition_adds_sliding_and_tumbling_per_option() { - let run = run_b(); - let summaries: BTreeSet<_> = options(&run, 0) - .into_iter() - .filter(|o| o != "exact") - .collect(); - let found: BTreeSet<_> = run - .logical - .iter() - .flat_map(windowed_builds) - .map(|(_, a, f, _)| (format!("{a:?}"), f)) - .collect(); - let forms = [ - WindowForm::None, - WindowForm::Sliding { - length_ms: PATTERN_B_WINDOW_MS, - slide_ms: PATTERN_B_INTERVAL_MS, - }, - WindowForm::Tumbling { - length_ms: PATTERN_B_INTERVAL_MS, - }, - ]; - for option in &summaries { - for form in forms { - assert!(found.contains(&(option.clone(), form)), "{option} {form:?}"); - } - } -} - -/// Each summary option of Pass 1 (KLL and DDSketch, both mergeable) also -/// comes as 1-min tumbling windows, gcd(5 min, 1 min), merged five at a time -/// before its p99 estimate; the whole-window builds are kept and read no -/// merge. Exact has no summary to split: 2 options × 2 forms + exact = 5. -#[test] -fn stage1_logical_asap_optimization_b_window_composition_adds_tumbling_per_mergeable_option() { - let run = run_b(); - assert_eq!(run.logical.len(), 5); - let mut found = BTreeSet::new(); - for c in &run.logical { - for (build, algorithm, form, _) in windowed_builds(c) { - let merged = estimates_of(&c.dag, build).iter().all(|(e, _)| { - c.dag - .producers(*e) - .into_iter() - .any(|p| is_merge(c.dag.payload(p))) - }); - assert_eq!(merged, form != WindowForm::None, "{}: {form:?}", c.id); - if let asap_types::ir::Operator::ASAP( - asap_types::ir::ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }, - ) = c.dag.payload(build) - { - assert_eq!(*tumbling_window_count, 5, "{}", c.id); - } - found.insert((format!("{algorithm:?}"), form)); - } - } - let tumbling = WindowForm::Tumbling { - length_ms: PATTERN_B_INTERVAL_MS, - }; - let expected: BTreeSet<_> = ["DDSketch", "Kll"] - .into_iter() - .flat_map(|a| [(a.to_string(), WindowForm::None), (a.to_string(), tumbling)]) - .collect(); - assert_eq!(found, expected); -} - -/// Window parameters are legal: L divides W, s divides L and the evaluation interval, a tumbling length divides both. -#[test] -fn stage1_logical_asap_optimization_b_window_parameters_are_legal() { - let tumbling = run_b() - .logical - .iter() - .flat_map(windowed_builds) - .filter(|(_, _, form, _)| matches!(form, WindowForm::Tumbling { .. })) - .count(); - assert!(tumbling > 0, "not vacuous"); - let (w, every) = (PATTERN_B_WINDOW_MS, PATTERN_B_INTERVAL_MS); - for c in &run_b().logical { - for (_, _, form, _) in windowed_builds(c) { - match form { - WindowForm::Sliding { - length_ms, - slide_ms, - } => { - assert_eq!(w % length_ms, 0, "{}: L | W", c.id); - assert_eq!(length_ms % slide_ms, 0, "{}: s | L", c.id); - assert_eq!(every % slide_ms, 0, "{}: s | interval", c.id); - } - WindowForm::Tumbling { length_ms } => { - assert_eq!(w % length_ms, 0, "{}: length | W", c.id); - assert_eq!(every % length_ms, 0, "{}: length | interval", c.id); - } - WindowForm::ExponentialHistogram { .. } => { - panic!("{}: Pattern B has no EH", c.id) - } - WindowForm::None => {} - } - } - } -} - -/// A sliding window with L = W reads one completed window (no merge); tumbling windows merge W / length of them. -#[test] -#[ignore = "needs Pass 2 window composition: sliding windows (#522); tumbling merges pass in \ - stage1_logical_asap_optimization_b_window_composition_adds_tumbling_per_mergeable_option"] -fn stage1_logical_asap_optimization_b_merge_only_where_the_window_form_needs_it() { - let run = run_b(); - let mut seen = BTreeSet::new(); - for c in &run.logical { - for (build, _, form, _) in windowed_builds(c) { - let merged = estimates_of(&c.dag, build).iter().any(|(e, _)| { - c.dag - .producers(*e) - .into_iter() - .any(|p| is_merge(c.dag.payload(p))) - }); - assert_eq!( - merged, - needs_merge(form, PATTERN_B_WINDOW_MS), - "{}: {form:?}", - c.id - ); - if form != WindowForm::None { - seen.insert(merged); - } - } - } - assert_eq!( - seen, - BTreeSet::from([false, true]), - "both a sliding and a tumbling candidate" - ); -} - -/// A window form that merges is used only with a summary whose states merge (KLL, DDSketch). -#[test] -fn stage1_logical_asap_optimization_window_merges_use_mergeable_summaries() { - for (run, window) in [(run_a(), 5 * YEAR_MS), (run_b(), PATTERN_B_WINDOW_MS)] { - for c in &run.logical { - for (_, algorithm, form, _) in windowed_builds(c) { - if needs_merge(form, window) { - assert!( - matches!(algorithm, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch), - "{}: {algorithm:?} {form:?}", - c.id - ); - } - } - } - } -} - -// ── Pattern B: plan selection ──────────────────────────────────────────── - -/// Plan selection selects one Pattern B candidate, explains the rest, and picks the cheapest valid one. -#[test] -fn stage3_plan_selection_b_selects_cheapest_valid() { - let run = run_b(); - assert_selects_one_and_explains_the_rest(&run); - assert_selects_cheapest_valid(&run); - assert_each_node_charged_per_consuming_query(&run); -} - -/// With nothing materialized, the 1-min KLL tumbling windows are all rebuilt at every -/// evaluation: they read the same 5 minutes of samples and update the same -/// states as one whole-window KLL, and merging five tumbling windows per series adds -/// 1M × 5 operations a minute. Each evaluation also holds its states while -/// it runs: the whole window's 1M KLLs, or the five tumbling windows' 5M smaller ones, -/// for the evaluation's latency. So whole-window KLL is selected at 2.363 per -/// second and the tumbling windows cost 3.611; exact costs 2.458. DDSketch, whole or in -/// tumbling windows, misses the rank target. -#[test] -fn stage3_plan_selection_b_tumbling_windows_rebuilt_at_query_time_add_the_merge() { - let run = run_b(); - let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); - let with = |algorithm: &str, tumbling_windows: bool| { - run.logical - .iter() - .find(|c| { - query_option(&c.dag, &c.query_roots, 0) == algorithm - && windowed_builds(c) - .iter() - .any(|(_, _, f, _)| (*f != WindowForm::None) == tumbling_windows) - }) - .unwrap_or_else(|| panic!("{algorithm} tumbling_windows={tumbling_windows}")) - }; - let whole = cost(with("Kll", false)).unwrap(); - let tumbling_windows = cost(with("Kll", true)).unwrap(); - assert!((whole - 2.362_500_025_2).abs() < 1e-9, "{whole}"); - assert!( - (tumbling_windows - 3.611_233_350_8).abs() < 1e-9, - "{tumbling_windows}" - ); - assert_eq!( - run.selection.selected, - run.physical_of(with("Kll", false)).next().unwrap().id - ); - for tumbling_windows in [false, true] { - assert_eq!( - cost(with("DDSketch", tumbling_windows)), - None, - "tumbling_windows={tumbling_windows}" - ); - } -} diff --git a/crates/integration-tests/tests/planner_layering_example4.rs b/crates/integration-tests/tests/planner_layering_example4.rs deleted file mode 100644 index a47c5fbfa..000000000 --- a/crates/integration-tests/tests/planner_layering_example4.rs +++ /dev/null @@ -1,509 +0,0 @@ -//! Rule-fixture and production acceptance for Example 4: materialization and objective models. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::planner_layering_common; - -use std::collections::{BTreeMap, BTreeSet}; - -use asap_cost::{ClusterLimits, ClusterObjective, CostInputs, CostModel, ResourceUnitCost}; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_types::ir::flat::NodeId; -use asap_types::ir::schema::SketchAlgorithm; -use asap_types::workload::{DataArrival, PlanningWorkload}; -use planner_layering_common::*; - -/// Pattern A as given: one ad hoc batch at T over mixed data. -fn once() -> PlanningWorkload { - pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ) -} - -fn monthly() -> PlanningWorkload { - pattern_a( - PatternARecurrence::MonthlyPredictable, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ) -} - -fn at_rest() -> PlanningWorkload { - pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::AtRest) -} - -/// The window-summary build of `c` with form `want` (`Some(form)` matches -/// exactly; `None` matches any EH) read by `readers` queries. -fn window_build(c: &Logical, want: Option, readers_count: usize) -> Option { - sketch_builds(&c.dag) - .into_iter() - .find_map(|(id, algorithm, _)| { - let form = window_form(&c.dag, id); - let matches = match want { - Some(want) => form == want, - None => matches!(form, WindowForm::ExponentialHistogram { .. }), - }; - (algorithm == SketchAlgorithm::Kll - && matches - && readers(&c.dag, &c.query_roots, id).len() == readers_count) - .then_some(id) - }) -} - -/// The physical candidates of the logical candidate with a KLL window -/// summary of `form` read by `readers_count` queries, keyed by the -/// materialization of that build. Several logical candidates may match (for -/// example with and without the shared input); the first is used. -fn options_of( - run: &Run, - form: Option, - readers_count: usize, -) -> BTreeMap { - let logical = run - .logical - .iter() - .find(|c| window_build(c, form, readers_count).is_some()) - .unwrap_or_else(|| { - let form = form.map_or("EH".to_string(), |f| format!("{f:?}")); - panic!("no logical candidate with a {form} KLL window summary") - }); - let build = window_build(logical, form, readers_count).unwrap(); - // Physical ASAP-aware optimization renumbers nodes; find the build again - // in each physical DAG. - run.physical_of(logical) - .map(|p| { - let id = sketch_builds(p.dag()) - .into_iter() - .find(|(id, a, _)| { - *a == SketchAlgorithm::Kll - && window_form(p.dag(), *id) == window_form(&logical.dag, build) - }) - .map(|(id, ..)| id) - .expect("the window summary survives physical ASAP-aware optimization"); - (materialization(p, id), (p.id.clone(), id)) - }) - .collect() -} - -fn eh_options(run: &Run) -> BTreeMap { - options_of(run, None, 5) -} - -fn tumbling() -> Option { - Some(WindowForm::Tumbling { - length_ms: PATTERN_B_INTERVAL_MS, - }) -} - -fn sliding() -> Option { - Some(WindowForm::Sliding { - length_ms: PATTERN_B_WINDOW_MS, - slide_ms: PATTERN_B_INTERVAL_MS, - }) -} - -fn cost(run: &Run, id: &str) -> f64 { - run.cost(id).unwrap_or_else(|| panic!("{id} is not priced")) -} - -use MaterializationOption::{MatAtIngestion, MatAtQuery, NotMaterialized}; - -// ── Constraints over every candidate ───────────────────────────────────── - -/// Every node upstream of an ingestion-time node also runs at ingestion time, in every workload variant. -/// The monthly variant runs in -/// `stage2_physical_asap_optimization_monthly_ingestion_time_upstream_is_ingestion_time`. -#[test] -fn stage2_physical_asap_optimization_ingestion_time_upstream_is_ingestion_time() { - for workload in [once(), at_rest(), pattern_b()] { - assert_ingestion_upstream_is_ingestion(&run_promql(&workload)); - } -} - -/// The monthly variant of -/// `stage2_physical_asap_optimization_ingestion_time_upstream_is_ingestion_time`: -/// it plans 67,228 candidates, too slow for a debug build in CI. -#[test] -#[ignore = "slow: run in release by hand"] -fn stage2_physical_asap_optimization_monthly_ingestion_time_upstream_is_ingestion_time() { - assert_ingestion_upstream_is_ingestion(&run_promql(&monthly())); -} - -/// Data at rest has no ingestion, so no candidate runs anything at ingestion time. -#[test] -fn stage2_physical_asap_optimization_at_rest_runs_nothing_at_ingestion_time() { - let run = run_promql(&at_rest()); - for p in &run.physical { - for n in &p.dag().nodes { - assert!(!runs_at_ingestion(p, n.id), "{}: {:?}", p.id, n.id); - } - } -} - -/// Each Pattern A query's range window and offset, as its PromQL text states them. -fn pattern_a_windows_and_offsets() -> (Vec, Vec) { - let windows = PATTERN_A.iter().map(|p| p.window_ms).collect(); - let offsets = PATTERN_A.iter().map(|p| p.offset_before_t_ms).collect(); - (windows, offsets) -} - -/// Every materialized output of every candidate of `workload` is kept for at -/// least its longest reader's `windows[q] + offsets[q]`. -fn assert_materialized_outputs_cover_their_consumers( - workload: &PlanningWorkload, - windows: &[u64], - offsets: &[u64], -) { - let run = run_promql(workload); - for p in &run.physical { - for n in &p.dag().nodes { - // A shared scan or range built once per evaluation holds its - // rows within that evaluation; plan selection reads it over the - // longest window above it. - let shared_rows = materialization(p, n.id) == MatAtQuery - && matches!( - relational(&n.payload).as_deref(), - Some("scan" | "time_range" | "time_shift") - ); - if materialization(p, n.id) == NotMaterialized || shared_rows { - continue; - } - let need = readers(p.dag(), &p.query_roots, n.id) - .into_iter() - .map(|q| windows[q] + offsets[q]) - .max() - .unwrap_or(0); - let kept = retention_ms(p, n.id).unwrap_or_else(|| { - panic!("{}: {:?} is materialized without a retention", p.id, n.id) - }); - assert!(kept >= need, "{}: {:?} kept {kept} < {need}", p.id, n.id); - } - } -} - -/// A materialized output is kept for as long as its consumers read: it covers every reader's lookback and offset. -/// The monthly variant runs in -/// `stage2_physical_asap_optimization_monthly_materialized_output_covers_its_consumers`. -#[test] -fn stage2_physical_asap_optimization_materialized_output_covers_its_consumers() { - let (windows, offsets) = pattern_a_windows_and_offsets(); - for workload in [once(), at_rest()] { - assert_materialized_outputs_cover_their_consumers(&workload, &windows, &offsets); - } - assert_materialized_outputs_cover_their_consumers(&pattern_b(), &[PATTERN_B_WINDOW_MS], &[0]); -} - -/// The monthly variant of -/// `stage2_physical_asap_optimization_materialized_output_covers_its_consumers`: -/// it plans 67,228 candidates, too slow for a debug build in CI. -#[test] -#[ignore = "slow: run in release by hand"] -fn stage2_physical_asap_optimization_monthly_materialized_output_covers_its_consumers() { - let (windows, offsets) = pattern_a_windows_and_offsets(); - assert_materialized_outputs_cover_their_consumers(&monthly(), &windows, &offsets); -} - -// ── Pattern A ──────────────────────────────────────────────────────────── - -/// The shared EH candidate yields A1 (query time, kept), A2 (ingestion time) and A3 (not materialized). -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage2_physical_asap_optimization_a_shared_eh_has_three_materialization_options() { - let found: BTreeSet<_> = eh_options(&run_promql(&once())).into_keys().collect(); - assert_eq!( - found, - BTreeSet::from([MatAtIngestion, MatAtQuery, NotMaterialized]) - ); -} - -/// With data at rest, A2 is not generated; A1 and A3 remain. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage2_physical_asap_optimization_a_at_rest_drops_the_ingestion_time_option() { - let found: BTreeSet<_> = eh_options(&run_promql(&at_rest())).into_keys().collect(); - assert_eq!(found, BTreeSet::from([MatAtQuery, NotMaterialized])); -} - -/// A materialized shared EH is one build node read by all five queries, charged once. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage2_physical_asap_optimization_a_materialized_eh_is_built_once_for_all_consumers() { - let run = run_promql(&once()); - for (m, (id, build)) in eh_options(&run) { - if m == NotMaterialized { - continue; - } - let p = run.physical(&id); - let ehs = sketch_builds(p.dag()) - .into_iter() - .filter(|(b, ..)| { - matches!( - window_form(p.dag(), *b), - WindowForm::ExponentialHistogram { .. } - ) - }) - .count(); - assert_eq!(ehs, 1, "{id}"); - assert_eq!(readers(p.dag(), &p.query_roots, build).len(), 5, "{id}"); - assert!( - run.selection.costs[&id].per_node.contains_key(&build), - "{id}" - ); - } -} - -/// A3 rebuilds the EH for each of the five queries (plan selection charges a -/// `NotMaterialized` node once per consuming query), so it costs more than A1, -/// which builds it once. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once() { - let run = run_promql(&once()); - let options = eh_options(&run); - let (a1, a3) = (&options[&MatAtQuery].0, &options[&NotMaterialized].0); - assert!( - cost(&run, a3) > cost(&run, a1), - "A3 {} vs A1 {}", - cost(&run, a3), - cost(&run, a1) - ); -} - -/// As given (run once, ad hoc), A1 is the cheapest of the three: A2 maintains years of history for one batch. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage3_plan_selection_a_once_adhoc_prefers_the_query_time_eh() { - let run = run_promql(&once()); - let options = eh_options(&run); - let a1 = cost(&run, &options[&MatAtQuery].0); - for (m, (id, _)) in &options { - assert!(a1 <= cost(&run, id), "A1 {a1} vs {m:?} {}", cost(&run, id)); - } -} - -/// Repeated monthly and predictable, A2's maintenance is shared by many batches, so A2 gains on A1. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage3_plan_selection_a_monthly_amortizes_ingestion_time_maintenance() { - let ratio = |workload: PlanningWorkload| { - let run = run_promql(&workload); - let options = eh_options(&run); - cost(&run, &options[&MatAtIngestion].0) / cost(&run, &options[&MatAtQuery].0) - }; - let (once, monthly) = (ratio(once()), ratio(monthly())); - assert!(monthly < once, "A2/A1: monthly {monthly} vs once {once}"); -} - -// ── Pattern B ──────────────────────────────────────────────────────────── - -/// A deployment that keeps query-time results across evaluations, which B3 -/// needs; the default deployment keeps none. -fn keeping_query_time_results() -> DeploymentCapabilities { - DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - } -} - -/// The 1-min tumbling KLL candidate yields B1 (ingestion time), B2 (not materialized) and B3 (query time, kept). -#[test] -fn stage2_physical_asap_optimization_b_tumbling_kll_has_three_materialization_options() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &keeping_query_time_results(), - ); - let found: BTreeSet<_> = options_of(&run, tumbling(), 1).into_keys().collect(); - assert_eq!( - found, - BTreeSet::from([MatAtIngestion, MatAtQuery, NotMaterialized]) - ); -} - -/// B1 builds the tumbling KLLs at ingestion time and merges and estimates at query time. -#[test] -fn stage2_physical_asap_optimization_b_b1_builds_at_ingestion_and_merges_at_query_time() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &DeploymentCapabilities::default(), - ); - let (id, build) = options_of(&run, tumbling(), 1)[&MatAtIngestion].clone(); - let p = run.physical(&id); - assert!(runs_at_ingestion(p, build)); - let estimates = estimates_of(p.dag(), build); - assert!(!estimates.is_empty()); - for (estimate, _) in estimates { - assert!( - !runs_at_ingestion(p, estimate), - "{id}: estimate at ingestion" - ); - let merge = p - .dag() - .producers(estimate) - .into_iter() - .find(|&m| is_merge(p.dag().payload(m))); - let merge = merge.unwrap_or_else(|| panic!("{id}: no merge before the estimate")); - assert!(!runs_at_ingestion(p, merge), "{id}: merge at ingestion"); - } -} - -/// Implementer test: for the default deployment, which keeps no query-time -/// result across evaluations, the tumbling KLL candidate yields B1 -/// (maintained at ingestion time) and B2 (rebuilt at every evaluation), not -/// B3; both are priced, and B2's evaluation stays within the 200 ms bound. -#[test] -fn stage2_physical_asap_optimization_b_tumbling_kll_has_b1_and_b2() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &DeploymentCapabilities::default(), - ); - let options = options_of(&run, tumbling(), 1); - assert_eq!( - options.keys().copied().collect::>(), - BTreeSet::from([MatAtIngestion, NotMaterialized]) - ); - for (id, _) in options.values() { - assert!(run.cost(id).is_some(), "{id} is priced"); - let latency = run.selection.costs[id].usage.latency_per_query[0].total_ms(); - assert!(latency <= 200.0, "{id}: {latency} ms"); - } -} - -/// B3 runs nothing at ingestion time; it keeps the tumbling KLLs it builds at query time. -/// Implementer addition: kept for the readers' 5-min look-back, it is -/// priced as building one 1-min tumbling window per evaluation, so it reads a fifth of -/// B2's raw data and keeps a fifth of it on disk. -#[test] -fn stage2_physical_asap_optimization_b_b3_keeps_query_time_windows() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &keeping_query_time_results(), - ); - let options = options_of(&run, tumbling(), 1); - let (id, build) = options[&MatAtQuery].clone(); - let p = run.physical(&id); - assert!( - p.dag().nodes.iter().all(|n| !runs_at_ingestion(p, n.id)), - "{id}" - ); - assert_eq!(retention_ms(p, build), Some(PATTERN_B_WINDOW_MS)); - let disk = |id: &str| run.selection.costs[id].usage.disk_bytes; - assert_eq!(disk(&id) * 5, disk(&options[&NotMaterialized].0)); -} - -/// The sliding-window KLL is kept from ingestion time or from query time; not materializing it is the no-window plan. -#[test] -#[ignore = "needs sliding windows (#522)"] -fn stage2_physical_asap_optimization_b_sliding_kll_has_two_materialization_options() { - let found: BTreeSet<_> = options_of( - &run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &DeploymentCapabilities::default(), - ), - sliding(), - 1, - ) - .into_keys() - .collect(); - assert_eq!(found, BTreeSet::from([MatAtIngestion, MatAtQuery])); -} - -// ── Cost models (#683, #660) ───────────────────────────────────────────── - -/// Pattern B under `cost_model`, with the default resource speed and horizon. -fn pattern_b_under(cost_model: CostModel) -> Run { - run_pattern_b_fixture( - &pattern_b(), - &CostInputs { - cost_model, - ..Default::default() - }, - &DeploymentCapabilities::default(), - ) -} - -/// The candidates the cost models choose between: the exact plan, the KLL -/// rebuilt over the whole window at query time and B1 (the tumbling windows maintained at ingestion time). -struct PatternBCandidates { - exact: String, - whole_window_kll: String, - b1: String, -} - -fn pattern_b_candidates(run: &Run) -> PatternBCandidates { - let tumbling_windows = options_of(run, tumbling(), 1); - PatternBCandidates { - exact: run - .physical - .iter() - .find(|p| sketch_builds(p.dag()).is_empty()) - .expect("an exact candidate") - .id - .clone(), - whole_window_kll: options_of(run, Some(WindowForm::None), 1)[&NotMaterialized] - .0 - .clone(), - b1: tumbling_windows[&MatAtIngestion].0.clone(), - } -} - -/// A cluster with `memory_gb` GB of memory and 1 TB of disk. -fn cluster(memory_gb: u64, objective: ClusterObjective) -> CostModel { - CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: memory_gb * 1_000_000_000, - disk_bytes: 1_000_000_000_000, - cpu_cores: None, - objective, - }) -} - -/// Each cost model chooses what its deployment pays for. Cost model 1 -/// (usage over time) picks the KLL rebuilt every minute: it uses memory only -/// while it runs, and B1 pays for 5.2 GB of tumbling windows every second. Cost model 2 -/// (provisioned capacity) picks the exact plan: every candidate needs one -/// core for its evaluation, and the exact plan's peak memory (192 MB of -/// per-series values) is the smallest, while the KLL's 1M states, each -/// grown for its 20 samples, take 1 GB at once. Cost model 3 (own cluster, -/// lowest latency) picks B1 when its 6.1 GB fit, since merging maintained -/// tumbling windows takes 7 ms against 126 ms for rebuilding; in 4 GB, B1 is invalid -/// and the KLL is fastest. -#[test] -fn stage3_plan_selection_b_cost_models_choose_differently() { - let usage_over_time = - pattern_b_under(CostModel::UsageOverTime(ResourceUnitCost::illustrative())); - let ids = pattern_b_candidates(&usage_over_time); - assert_eq!(usage_over_time.selection.selected, ids.whole_window_kll); - - let provisioned = pattern_b_under(CostModel::ProvisionedCapacity( - ResourceUnitCost::illustrative(), - )); - assert_eq!(provisioned.selection.selected, ids.exact); - let peak = |run: &Run, id: &str| run.selection.costs[id].usage.peak_memory_bytes; - assert!(peak(&provisioned, &ids.exact) < peak(&provisioned, &ids.whole_window_kll)); - - let large = pattern_b_under(cluster(256, ClusterObjective::LowestLatency)); - assert_eq!(large.selection.selected, ids.b1); - assert!(large - .selection - .costs - .values() - .all(|c| c.cost_per_second.is_none())); - - let small = pattern_b_under(cluster(4, ClusterObjective::LowestLatency)); - assert_eq!(small.selection.selected, ids.whole_window_kll); - let invalid = small.invalid(); - let b1 = invalid[ids.b1.as_str()]; - assert!(b1.starts_with("peak memory"), "{b1}"); -} diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs deleted file mode 100644 index 9db3c4947..000000000 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ /dev/null @@ -1,641 +0,0 @@ -//! Planner-selected summaries over raw data samples compile, bound to raw data -//! at their `Scan`, and produce the same estimates as feeding their kernel -//! sample by sample. -use crate::physical_common; -use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; -use asap_types::ir::ASAPOp; -use asap_types::ir::OperatorNode; -use physical_common::compile_at_ingestion_physical_asap_dag; -use std::{collections::BTreeMap, collections::BTreeSet, rc::Rc, sync::Arc}; - -use asap_executor::{ - factory::create_planner_accumulator, - physical_planner::{compile, promql_rows, CompiledPhysicalDAG}, - runtime::{Limits, RunContext, Scope}, - summary_kernels::{exact::ExactAccumulator, weighted_frequency::WeightedFrequency}, - values::Value, - AggregateCore, -}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::explicit_local_candidates; -use asap_types::ir::operator::Reduction; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::expr_ir::ColumnRef; -use asap_types::ir::schema::{ - FieldDataType, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, -}; -use futures::{executor::block_on, StreamExt}; - -type Series = BTreeMap; - -/// A series label set; `None` omits the label. An empty value is present -/// in the input but is not part of the series identity. -fn series(service: Option<&str>, instance: &str) -> Series { - [ - ("__name__", Some("m")), - ("service", service), - ("instance", Some(instance)), - ] - .into_iter() - .filter_map(|(k, v)| Some((k.to_owned(), v?.to_owned()))) - .collect() -} - -fn canonical(labels: &Series) -> Series { - labels - .iter() - .filter(|(_, v)| !v.is_empty()) - .map(|(k, v)| (k.clone(), v.clone())) - .collect() -} - -/// Exact and explicit single-rule candidates for `query`, with -/// `m` declared a counter: plan selection's answer to a heap's open -/// requirement that its weights are counter samples, which the executor -/// needs before it builds a Count-Min heap. -fn candidates(query: &str, accuracy: AccuracyTarget) -> Vec> { - use asap_types::ir::QueryRoot; - use asap_types::workload::bounds::with_confirmed_weights; - use asap_types::workload::{DataWorkload, PrometheusMetricType, SourceFacts}; - let root = lower_promql(query, accuracy.clone()).expect("lowering failed"); - let root = promql_rows::with_series_identity(&root).unwrap(); - let counter = SourceFacts { - metric_type: Some(PrometheusMetricType::Counter), - ..SourceFacts::default() - }; - let data = DataWorkload { - sources: [("m".to_string(), counter)].into(), - ..Default::default() - }; - explicit_local_candidates(root) - .into_iter() - .map(|candidate| { - match &with_confirmed_weights(&[QueryRoot::Operator(candidate)], &data)[..] { - [QueryRoot::Operator(confirmed)] => Rc::clone(confirmed), - _ => unreachable!("one operator root"), - } - }) - .collect() -} - -/// Raw-input summary nodes: `(dag, raw source id, summary id)`. -fn raw_summaries(dag: &PhysicalASAPDAG) -> Vec<(u64, u64)> { - dag.nodes - .iter() - .filter(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { .. }) - ) - }) - .filter_map(|node| { - let inputs = dag - .edges - .iter() - .filter(|edge| edge.consumer == node.id) - .collect::>(); - let [edge] = inputs.as_slice() else { - return None; - }; - let source = dag.nodes.iter().find(|n| n.id == edge.producer)?; - matches!( - source.payload, - PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::TimeRange { .. }) - ) - .then_some((source.id as u64, node.id as u64)) - }) - .collect() -} - -fn samples() -> Vec<(Series, i64, f64)> { - let mut rows = Vec::new(); - let series_set = [ - (Some("a"), "1"), - (Some("a"), "2"), - (Some("b"), "1"), - (None, "3"), - (Some("b"), ""), - ]; - for (index, (service, instance)) in series_set.iter().enumerate() { - for step in 1..=5i64 { - let value = (index as f64 + 1.0) * step as f64 + (step % 2) as f64; - rows.push((series(*service, instance), step * 1000, value)); - } - } - rows -} - -/// Compile the summary `root` alone, its raw data read at the `Scan`. -fn compile_summary(dag: &PhysicalASAPDAG, root: u64) -> Result { - compile(dag, &[], &[root]).map_err(|error| error.to_string()) -} - -/// Run the summary `root` in one ingestion run over `rows`, bound as raw -/// data at its `Scan`: each population's labels and state. -fn execute( - dag: &PhysicalASAPDAG, - root: u64, - rows: &[(Series, i64, f64)], -) -> Vec<(Series, Arc)> { - let program = compile_summary(dag, root).unwrap_or_else(|error| { - panic!("raw summary {root} does not compile: {error}"); - }); - let store = physical_common::raw_data_store(dag, |_, schema| { - rows.iter() - .map(|(labels, time, value)| { - promql_rows::series_row(schema, &canonical(labels), *time, *value).unwrap() - }) - .collect() - }); - let output = program.output_schema(root).unwrap(); - let physical_dag = program.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 6000, - }, - Limits::default(), - ) - .unwrap(); - block_on(async { - let mut stream = physical_dag - .execute(program.roots(), context) - .unwrap() - .remove(0); - let mut result = Vec::new(); - while let Some(batch) = stream.next().await { - for row in &batch.unwrap().to_rows() { - let mut labels = Series::new(); - let mut state = None; - for (field, value) in output.fields.iter().zip(row) { - match value { - Value::Utf8(identity) - if field.name == promql_rows::SERIES_IDENTITY_COLUMN => - { - labels.extend(promql_rows::decode_series_identity(identity).unwrap()) - } - Value::Utf8(label) => { - labels.insert(field.name.clone(), label.to_string()); - } - Value::Summary { state: s, .. } => state = Some(s.clone()), - Value::Timestamp(6000) => {} - other => panic!("unexpected population column {other:?} in {row:?}"), - } - } - result.push((canonical(&labels), state.expect("a summary state"))); - } - } - result - }) -} - -/// Each series' latest sample, moved to the ingestion run's end (6000 ms). -fn latest_per_series(rows: &[(Series, i64, f64)]) -> Vec<(Series, i64, f64)> { - let mut latest = BTreeMap::::new(); - for (labels, time, value) in rows { - let entry = latest.entry(canonical(labels)).or_insert((*time, *value)); - if *time >= entry.0 { - *entry = (*time, *value); - } - } - latest - .into_iter() - .map(|(labels, (_, value))| (labels, 6000, value)) - .collect() -} - -/// PromQL grouping of a canonical label set: `by` keeps the named labels, -/// `without` drops them and `__name__`. -fn population(reduction: &Reduction, dag_labels: &[String], labels: &Series) -> Series { - let labels = canonical(labels); - match reduction { - Reduction::PerEntity => labels, - Reduction::Reduce(keys) => labels - .into_iter() - .filter(|(k, _)| { - if keys.is_without() { - k != "__name__" && !dag_labels.contains(k) - } else { - dag_labels.contains(k) - } - }) - .collect(), - } -} - -/// Item identity used by a keyed summary: labels by name, the sample value, -/// or the canonical label-set identity. -fn item(expr: &SummaryInputExpr, labels: &Series, value: f64, out: &mut Vec) { - match expr { - SummaryInputExpr::Column(ColumnRef::SampleValue) => out.push(Value::Float64(value)), - SummaryInputExpr::Column(ColumnRef::Named(name)) if name == "value" => { - out.push(Value::Float64(value)) - } - SummaryInputExpr::Column(ColumnRef::Named(name)) - if name == promql_rows::SERIES_IDENTITY_COLUMN => - { - out.push(Value::Utf8( - promql_rows::encode_series_identity(&canonical(labels)) - .unwrap() - .into(), - )) - } - SummaryInputExpr::Column(ColumnRef::Named(name)) => out.push(Value::Utf8( - canonical(labels) - .get(name) - .cloned() - .unwrap_or_default() - .into(), - )), - SummaryInputExpr::Tuple(items) => items.iter().for_each(|i| item(i, labels, value, out)), - other => panic!("unsupported fixture item {other:?}"), - } -} - -fn weight(update: &SummaryUpdate, value: f64) -> f64 { - match update.weight { - SummaryInputExpr::Constant(weight) => weight, - _ => value, - } -} - -/// Estimates that identify a state's content for comparison. -fn evaluations(state: &dyn AggregateCore, family: &FieldDataType) -> Vec { - if let Some(exact) = state.as_any().downcast_ref::() { - let FieldDataType::ExactAggregate(kind, _) = family else { - unreachable!() - }; - return vec![exact.evaluation(*kind, None, None).unwrap().unwrap()]; - } - let FieldDataType::Sketch(kind, _) = family else { - panic!("sketch state for exact family") - }; - match kind.algorithm() { - SketchAlgorithm::Kll | SketchAlgorithm::DDSketch => [0.1, 0.5, 0.9] - .into_iter() - .map(|q| state.estimate(&SketchStatistic::Quantile { q }).unwrap()) - .collect(), - SketchAlgorithm::Hll | SketchAlgorithm::Kmv => { - vec![state.estimate(&SketchStatistic::Cardinality).unwrap()] - } - SketchAlgorithm::UnivMon => [ - SketchStatistic::Cardinality, - SketchStatistic::FrequencyL2, - SketchStatistic::FrequencyEntropy, - ] - .iter() - .map(|statistic| state.estimate(statistic).unwrap()) - .collect(), - other => panic!("unexpected unkeyed sketch {other:?}"), - } -} - -/// Compile one raw-input summary, execute it over `rows`, and compare each -/// population with its kernel fed sample by sample. Returns the family label, -/// or the family when it has no native state. -fn check( - query: &str, - dag: &PhysicalASAPDAG, - source: u64, - root: u64, - rows: &[(Series, i64, f64)], -) -> Result { - let node = dag.nodes.iter().find(|n| n.id as u64 == root).unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family, - input, - reduction, - grouping, - .. - }) = &node.payload - else { - unreachable!() - }; - let source_node = dag.nodes.iter().find(|n| n.id as u64 == source).unwrap(); - // An instant selector keeps each series' latest sample within its - // lookback before the run's end: feed one such sample per series. - let latest_rows; - let rows = if matches!( - source_node.payload, - PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::TimeRange { - kind: asap_types::ir::TimeRangeKind::Instant, - .. - }) - ) { - latest_rows = latest_per_series(rows); - &latest_rows[..] - } else { - rows - }; - let keys = match reduction { - Reduction::Reduce(keys) => keys - .keys() - .iter() - .map(|i| source_node.output_schema.fields[*i].name.clone()) - .collect(), - Reduction::PerEntity => vec![], - }; - let stored_only = matches!(family, FieldDataType::Sketch(kind, _) - if kind.algorithm() == &asap_types::ir::schema::SketchAlgorithm::Cms); - if stored_only || asap_executor::capability::validate_native_family(family).is_err() { - // Families without a native state, or with native stored state only (plain CMS), are outside raw summary execution; - // their compile must fail. - assert!(compile_summary(dag, root).is_err()); - return Err(format!("{family:?}")); - } - let actual = execute(dag, root, rows); - let label = match family { - FieldDataType::ExactAggregate(kind, _) => format!("{kind:?}"), - FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), - other => format!("{other:?}"), - }; - if let FieldDataType::Sketch(kind, _) = family { - if let (Some(keyed), false) = ( - &input.item, - matches!( - kind.algorithm(), - SketchAlgorithm::Hll | SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon - ), - ) { - // Keyed heaps: every item's estimated weight is its exact - // total at this scale (no collisions in the fixture). - let mut expected = BTreeMap::>::new(); - for (labels, _, value) in rows { - let mut items = Vec::new(); - item(keyed, labels, *value, &mut items); - *expected - .entry(population(reduction, &keys, labels)) - .or_default() - .entry(format!("{items:?}")) - .or_default() += weight(input, *value); - } - assert_eq!(actual.len(), expected.len(), "{query}"); - for (labels, state) in &actual { - let heap = state.as_any().downcast_ref::().unwrap(); - let got = heap - .rows(usize::MAX >> 1) - .into_iter() - .map(|mut row| { - let Some(Value::Float64(score)) = row.pop() else { - panic!("heap score") - }; - (format!("{row:?}"), score) - }) - .collect::>(); - assert_eq!(&got, &expected[labels], "{query}"); - } - return Ok(label); - } - } - let mut expected = - BTreeMap::>::new(); - for (labels, time, value) in rows { - let updater = expected - .entry(population(reduction, &keys, labels)) - .or_insert_with(|| create_planner_accumulator(family, input, grouping).unwrap()); - let unit = input.item.is_some(); - updater - .update_single(if unit { *value } else { weight(input, *value) }, *time) - .unwrap(); - } - assert_eq!(actual.len(), expected.len(), "{query}"); - for (labels, state) in actual { - let reference = expected[&labels].snapshot_accumulator(); - assert_eq!( - evaluations(state.as_ref(), family), - evaluations(reference.as_ref(), family), - "{query}: {labels:?}" - ); - } - Ok(label) -} - -// Every raw-input summary Planner composes compiles over raw data sample rows, -// and each population's estimates equal feeding its kernel sample by sample. -#[test] -fn raw_data_sample_summaries_compile_and_match_their_kernels() { - let exact = AccuracyTarget::exact(); - let sketch = AccuracyTarget::max_error(0.02); - // A UnivMon sized for a distinct count at ε = 0.02 holds a heap of - // millions of keys on each of its 19 layers, past the default 64 MiB. - let distinct = AccuracyTarget::max_error(0.2).failure_probability(0.1); - let queries = [ - ("sum_over_time(m[5m])", &exact), - ("count_over_time(m[5m])", &exact), - ("min_over_time(m[5m])", &exact), - ("max_over_time(m[5m])", &exact), - ("rate(m[5m])", &exact), - ("increase(m[5m])", &exact), - ("sum by (service) (sum_over_time(m[5m]))", &exact), - ("sum by (service) (rate(m[5m]))", &exact), - ("topk(2, sum_over_time(m[5m]))", &exact), - ("quantile_over_time(0.9, m[5m])", &sketch), - ("sum by (service) (quantile_over_time(0.9, m[5m]))", &sketch), - ("quantile by (service) (0.9, m)", &sketch), - ("distinct_over_time(m[5m])", &distinct), - ("count(m)", &sketch), - ("topk(2, m)", &sketch), - ( - "topk(2, sum by (service) (count_over_time(m[5m])))", - &sketch, - ), - ("topk(2, sum_over_time(m[5m]))", &sketch), - ("topk by (service) (2, sum_over_time(m[5m]))", &sketch), - ]; - let rows = samples(); - let mut families = BTreeSet::new(); - let mut unsupported = BTreeSet::new(); - let mut checked = BTreeMap::new(); - for (query, accuracy) in queries { - for candidate in candidates(query, accuracy.clone()) { - let dag = compile_at_ingestion_physical_asap_dag(&candidate).unwrap(); - for (source, root) in raw_summaries(&dag) { - match check(query, &dag, source, root, &rows) { - Ok(family) => { - families.insert(family); - *checked.entry(query).or_insert(0) += 1; - } - Err(family) => { - unsupported.insert(family); - } - } - } - } - } - println!("checked {checked:?}; families {families:?}; without native state {unsupported:?}"); - for query in [ - "topk by (service) (2, sum_over_time(m[5m]))", - "quantile by (service) (0.9, m)", - "distinct_over_time(m[5m])", - ] { - assert!( - checked.contains_key(query), - "{query} has no checked raw summary" - ); - } - for family in [ - "Sum", - "Count", - "Min", - "Max", - "Rate", - "Increase", - "Kll", - "DDSketch", - "Hll", - "Kmv", - "UnivMon", - // The whole-expression heaps over raw data samples. - "CmsWithHeap", - "CountSketchWithHeap", - ] { - assert!( - families.contains(family), - "no {family} fixture: {families:?}" - ); - } -} - -/// Replace the raw summary of `sum by (service) (sum_over_time(m{instance!="none"}[5m]))` with -/// another update, keeping its raw input and reduction. -fn grouped_raw_summary(family: FieldDataType, input: SummaryUpdate) -> (PhysicalASAPDAG, u64, u64) { - // The matcher, true of every sample, makes the `Scan` read `instance`. - let candidate = candidates( - r#"sum by (service) (sum_over_time(m{instance!="none"}[5m]))"#, - AccuracyTarget::exact(), - ) - .pop() - .unwrap(); - let mut dag = compile_at_ingestion_physical_asap_dag(&candidate).unwrap(); - let (source, root) = raw_summaries(&dag)[0]; - let node = dag.nodes.iter_mut().find(|n| n.id as u64 == root).unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: old, - input: update, - .. - }) = &mut node.payload - else { - unreachable!() - }; - for field in &mut node.output_schema.fields { - if field.dtype == *old { - field.dtype = family.clone(); - } - } - *old = family; - *update = input; - let schema = node.output_schema.clone(); - for edge in dag.edges.iter_mut().filter(|e| e.producer as u64 == root) { - *edge.intermediate_schema_mut() = schema.clone(); - } - (dag, source, root) -} - -// Keyed heaps over raw data samples resolve items from the series label set and -// estimate each item's exact total; invalid weight contracts do not compile. -#[test] -fn raw_data_sample_heaps_resolve_items_from_labels() { - use asap_types::ir::schema::GroupingStrategy::PerSubpopulationInstance; - use asap_types::ir::schema::{NonNegativeWeightProof, SketchKind, SketchParams, WeightDomain}; - let heap = |algorithm, params| { - FieldDataType::Sketch(SketchKind::new(algorithm, params), PerSubpopulationInstance) - }; - let cms = heap( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 3, - heap_size: 8, - }, - ); - let count_sketch = heap( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 64, - depth: 3, - heap_size: 8, - }, - ); - // The series label set, read from the `Scan`'s series identity column. - let identity = SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::Named( - promql_rows::SERIES_IDENTITY_COLUMN.into(), - ))), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }; - let by_instance = SummaryUpdate { - item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( - ColumnRef::Named("instance".into()), - )])), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::UnknownOrSigned, - }; - let rows = samples(); - for (family, input) in [ - (cms.clone(), identity.clone()), - (count_sketch.clone(), identity), - (count_sketch, by_instance.clone()), - ] { - let unit_count = input.weight == SummaryInputExpr::Constant(1.0); - let (dag, source, root) = grouped_raw_summary(family, input); - assert!(check("heap", &dag, source, root, &rows).is_ok()); - // A unit count adds the constant 1 per row; no column of ones is - // projected for it. - let compiled = serde_json::to_value(compile_summary(&dag, root).unwrap()).unwrap(); - assert_eq!( - compiled - .to_string() - .contains(r#""row_weight":{"Constant":1}"#), - unit_count, - "{compiled}" - ); - } - let signed_cms = grouped_raw_summary(cms, by_instance.clone()); - assert!(compile_summary(&signed_cms.0, signed_cms.2).is_err()); -} - -// End to end: `topk(2, count by (service) (m))` at ε = 0.02 has a Count-Min -// heap candidate whose update is the unit count (`Constant(1.0)`). Placed at -// ingestion, its raw-data-sample precompute adds the constant 1 per sample to each -// service's count, with no column of ones projected, and each heap reads the -// exact per-service sample counts of the fixture. -#[test] -fn count_ranked_heap_precompute_adds_a_constant_row_weight() { - let query = "topk(2, count by (service) (m))"; - let rows = samples(); - let mut checked = 0; - for candidate in candidates(query, AccuracyTarget::max_error(0.02)) { - let dag = compile_at_ingestion_physical_asap_dag(&candidate).unwrap(); - for (source, root) in raw_summaries(&dag) { - let node = dag.nodes.iter().find(|n| n.id as u64 == root).unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - input, - .. - }) = &node.payload - else { - continue; - }; - if kind.algorithm() != &SketchAlgorithm::CmsWithHeap { - continue; - } - assert_eq!(input.weight, SummaryInputExpr::Constant(1.0)); - let compiled = serde_json::to_value(compile_summary(&dag, root).unwrap()) - .unwrap() - .to_string(); - assert!( - compiled.contains(r#""row_weight":{"Constant":1}"#), - "{compiled}" - ); - assert_eq!( - check(query, &dag, source, root, &rows), - Ok("CmsWithHeap".into()) - ); - checked += 1; - } - } - assert!(checked > 0, "{query} has no Count-Min heap candidate"); -} diff --git a/crates/integration-tests/tests/production_rollup_workloads.rs b/crates/integration-tests/tests/production_rollup_workloads.rs index c44f7724a..9273f878e 100644 --- a/crates/integration-tests/tests/production_rollup_workloads.rs +++ b/crates/integration-tests/tests/production_rollup_workloads.rs @@ -9,7 +9,7 @@ use asap_executor::physical_planner::promql_rows; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::fixtures::lower_promql; use asap_planner::{run_planning, PlanSearchSettings, PlanningModels, PlanningRun}; -use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; +use asap_types::accuracy::{AccuracyMetric, AccuracyScope, AccuracyTarget}; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; use asap_types::workload::{ @@ -57,7 +57,7 @@ fn data() -> DataWorkload { "sources": {"http": { "series_count": 10_000, "sample_interval": 5, - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "max_distinct_per_evaluation": 1_000_000, "rows_held": 120_000_000, }}, @@ -84,42 +84,23 @@ fn production( .unwrap() } -fn check( - run: &PlanningRun, - queries: usize, - expected_rollups: usize, - family: &str, - separate_cost: Option, -) { - let selection = run +fn check(run: &PlanningRun, queries: usize, expected_rollups: usize, family: &str) { + let (plan, selection) = run .selection .as_ref() .expect("validated production plan within 30s"); - println!("selected cost={:?}", selection.cost.cost_per_second); - let selected_cost = selection.cost.cost_per_second.unwrap(); - if let Some(separate) = separate_cost { - // Both searches must improve on independently built KLLs at the - // same accuracy target. - assert!(selected_cost < separate); - } let accepted = &selection.accepted_accuracy_per_query; assert_eq!(accepted.len(), queries); - assert!(accepted.iter().all(|a| !a.accepted_regardless_of_guarantee)); - if separate_cost.is_some() && accepted.iter().all(|a| a.sketches.is_empty()) { - println!("exact plan costs less than both the direct incumbent and independent KLLs"); - return; - } assert!( accepted .iter() - .all(|a| !a.accepted_regardless_of_guarantee && a.sketches.iter().any(|s| s == family)), + .all(|a| a.sketches.iter().any(|s| s == family)), "{accepted:?}" ); - let selected = &run.selected_physical_plan.as_ref().unwrap().plan; let mut seen = HashSet::new(); let mut builds = 0; let mut rollups = 0; - for root in &selected.roots { + for root in &plan.query_roots().unwrap() { let QueryRoot::Operator(root) = root else { panic!("operator output") }; @@ -149,9 +130,7 @@ async fn sql(queries: &[String], accuracy: AccuracyTarget) -> PlanningRun { let mut roots = Vec::new(); for query in queries { roots.push(QueryRoot::Operator( - lower_sql(query, &catalog(), accuracy.clone()) - .await - .unwrap(), + lower_sql(query, &catalog()).await.unwrap(), )); } production( @@ -168,7 +147,7 @@ async fn production_three_sql_groupings_share_kll() { .accuracy_metric(AccuracyMetric::Rank) .accuracy_scope(AccuracyScope::PerValue); let queries: Vec<_> = ["region, service", "region", "service"].iter().map(|keys| format!("SELECT {keys}, approx_percentile_cont(latency, 0.99) AS value FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY {keys}")).collect(); - check(&sql(&queries, accuracy).await, 3, 2, "Kll", None); + check(&sql(&queries, accuracy).await, 3, 2, "Kll"); } #[tokio::test] @@ -178,97 +157,7 @@ async fn production_sql_distinct_region_service_rollup() { .accuracy_metric(AccuracyMetric::Relative) .accuracy_scope(AccuracyScope::PerValue); let queries: Vec<_> = ["region, service", "service"].iter().map(|keys| format!("SELECT {keys}, COUNT(DISTINCT user_id) AS value FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY {keys}")).collect(); - check(&sql(&queries, accuracy).await, 2, 1, "Kmv", None); -} - -#[test] -// The earlier controlled fixture omitted raw retention. This production -// workload explicitly holds one current sample for each of its 10,000 series. -fn production_promql_compares_rollup_with_exact() { - let accuracy = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank); - let roots: Vec<_> = [ - "quantile by (host, job) (0.5, data)", - "quantile by (job) (0.5, data)", - ] - .iter() - .map(|query| { - let root = lower_promql(query, accuracy.clone()).unwrap(); - QueryRoot::Operator(promql_rows::with_series_identity(&root).unwrap()) - }) - .collect(); - let data: DataWorkload = serde_json::from_value(serde_json::json!({"arrival":{"ingesting":{"rows_per_second":666.67}},"sources":{"data":{"series_count":10000,"metric_type":"gauge","sample_interval":15000,"rows_held":10000}}})).unwrap(); - let entries = [entry(&accuracy, 10000), entry(&accuracy, 10000)]; - let separate = independent_kll_cost(roots.clone(), &entries, &data); - println!("independent KLL cost={separate}"); - check( - &production(roots, &entries, &data), - 2, - 1, - "Kll", - Some(separate), - ); -} - -// Price the same workload with independently built, unmaterialized KLL states. -fn independent_kll_cost( - roots: Vec, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, -) -> f64 { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let choices: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - HowToComputeAggregateMeasure::Summary( - asap_types::ir::schema::SketchAlgorithm::Kll - ) - ) - }) - .unwrap() - }) - .collect(); - let roots: Vec<_> = compose_logical_candidate(&inventory, &choices).unwrap(); - // Composed outside the memo: sized as Pass 1 sizes a candidate. - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let roots = - asap_accuracy::sizing::with_summaries_sized_for_targets(&roots, &targets, data).unwrap(); - let mut settings = CostInputs::default(); - settings.cost_model = CostModel::UsageOverTime(ResourceUnitCost::aws_fargate_us_east_1( - settings.resource_speed, - )); - let models = PlanningModels::builtin().with_cost_inputs(&settings); - let plan = asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - "independent", - &roots, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - &mut |_, _, _| Some(asap_types::physical::Materialization::NotMaterialized), - ) - .unwrap() - .unwrap(); - let result = - asap_devtools::memo_pricing::reference_select(&[plan], entries, data, models).unwrap(); - result.costs[&result.selected].cost_per_second.unwrap() + check(&sql(&queries, accuracy).await, 2, 1, "Kmv"); } // Exercise multi-input operators through the production route under more than @@ -303,15 +192,6 @@ fn production_binary_queries_use_joint_prices_with_mixed_schedules() { })) .unwrap(); let run = production(roots, &entries, &data); - let selection = run.selection.as_ref().expect("validated binary workload"); - println!("mixed binary workload report={:?}", selection.search_report); - let accepted = &selection.accepted_accuracy_per_query; - assert_eq!(accepted.len(), queries.len()); - assert!(accepted - .iter() - .all(|item| !item.accepted_regardless_of_guarantee)); - assert!(selection - .cost - .cost_per_second - .is_some_and(|cost| cost.is_finite() && cost >= 0.0)); + let (_, selection) = run.selection.as_ref().expect("validated binary workload"); + assert_eq!(selection.accepted_accuracy_per_query.len(), queries.len()); } diff --git a/crates/integration-tests/tests/promql_numeric_regressions.rs b/crates/integration-tests/tests/promql_numeric_regressions.rs deleted file mode 100644 index 0e8469f03..000000000 --- a/crates/integration-tests/tests/promql_numeric_regressions.rs +++ /dev/null @@ -1,344 +0,0 @@ -//! Numeric regression fixtures: actual PromQL lowering plus numeric update/evaluation checks. -//! The count/sum interpreter below verifies planner update semantics, not a deployed backend. -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::physical_export::physical_asap_dag; -use asap_integration_tests::planning::{explicit_local_candidates, explicit_summary_candidate}; -use asap_types::ir::operator::Reduction; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ExactKind, FieldDataType, SummaryInputExpr, SummaryUpdate}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use std::rc::Rc; - -/// The first explicitly generated local-rule fixture containing an ASAP -/// operator, or its exact computation when no summary is applicable. -fn plan(query: &str, accuracy: AccuracyTarget) -> Rc { - let pre = lower_promql(query, accuracy.clone()).unwrap(); - let candidates = explicit_local_candidates(pre); - candidates - .iter() - .find(|node| node.contains_asap()) - .unwrap_or(&candidates[0]) - .clone() -} -fn aggregate(node: &OperatorNode) -> (&FieldDataType, &SummaryUpdate, &Reduction) { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { - family, - input, - reduction, - .. - }) => (family, input, reduction), - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => aggregate(summary_input), - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => aggregate(child), - // A value operation (Project/Filter/Sort/Limit/...) over the state. - Operator::NonASAP(_) if node.children().len() == 1 && node.contains_asap() => { - aggregate(node.children()[0]) - } - other => panic!("not a maintained accumulator: {other:?}"), - } -} -fn contribution(family: &FieldDataType, update: &SummaryUpdate, value: f64) -> f64 { - if matches!(family, FieldDataType::ExactAggregate(ExactKind::Count, _)) { - return 1.; - } - match update.weight { - SummaryInputExpr::Constant(v) => v, - SummaryInputExpr::Column(ColumnRef::SampleValue) => value, - ref other => panic!("unexpected update {other:?}"), - } -} - -/// Target count depends on series multiplicity, never distinct values or health. -#[test] -fn count_up_counts_targets_even_when_values_repeat_or_change_sign() { - let node = plan("count(up)", AccuracyTarget::exact()); - let (family, update, _) = aggregate(&node); - assert!(matches!( - family, - FieldDataType::ExactAggregate(ExactKind::Count, _) - )); - for values in [[1., 1., 1.], [1., 1., 0.], [0., 0., 0.], [-1., -1., -1.]] { - assert_eq!( - values - .into_iter() - .map(|v| contribution(family, update, v)) - .sum::(), - 3. - ); - } - physical_asap_dag(&node); -} - -/// Ten samples give count ten, whereas sum retains the signed sample values. -#[test] -fn window_counts_and_sums_distinguish_one_zero_three_and_negative_values() { - for (query, kind, is_count) in [ - ("count_over_time(up[5m])", ExactKind::Count, true), - ("sum_over_time(up[5m])", ExactKind::Sum, false), - ] { - let node = plan(query, AccuracyTarget::exact()); - let (family, update, reduction) = aggregate(&node); - assert!(matches!(family, FieldDataType::ExactAggregate(k, _) if *k == kind)); - assert_eq!(*reduction, Reduction::PerEntity); - for value in [1., 0., 3., -3.] { - let got: f64 = (0..10).map(|_| contribution(family, update, value)).sum(); - assert_eq!(got, if is_count { 10. } else { value * 10. }); - } - physical_asap_dag(&node); - } -} - -/// Exact summary candidates must exist, rather than merely retaining the original query. -#[test] -fn sum_rate_and_increase_have_real_exact_accumulator_nodes() { - for (query, kind) in [ - ("sum by(job)(up)", ExactKind::Sum), - ("rate(requests_total[5m])", ExactKind::Rate), - ("increase(requests_total[5m])", ExactKind::Increase), - ] { - let node = plan(query, AccuracyTarget::exact()); - let (family, _, _) = aggregate(&node); - assert!(matches!(family, FieldDataType::ExactAggregate(k, _) if *k == kind)); - assert!(node.guarantee.as_ref().unwrap().is_exact()); - physical_asap_dag(&node); - } -} - -/// The PromQL quantile over a DDSketch: the linear interpolation at rank -/// `q (n - 1)` between the sketch's two neighbouring order statistics. -fn interpolated_quantile(sketch: &asap_sketchlib::DdSketch, q: f64) -> f64 { - sketch.quantile_interpolated(q).unwrap().unwrap() -} - -/// A finite, nonzero approximate denominator does not certify a ratio bound. -#[test] -fn checked_ratio_must_not_certify_cross_zero_interpolation() { - let root = lower_promql( - "quantile_over_time(0.5, data[5m]) / quantile_over_time(0.9, data[5m])", - AccuracyTarget::max_error(0.01), - ) - .unwrap(); - let node = explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::DDSketch); - assert_eq!(OperatorNode::reachable(&node).iter().filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) if *kind.algorithm() == asap_types::ir::schema::SketchAlgorithm::DDSketch)).count(), 2, - "the signed interpolation counterexample requires two explicit DDSketch builds"); - assert!( - matches!(node.operator, Operator::NonASAP(NonASAPOp::BinaryOp { .. })) - && node.contains_asap(), - "direct quantile ratio should remain an available candidate" - ); - assert!(node.guarantee.is_none(), "{:?}", node.guarantee); - use asap_accuracy::propagation::bind_guarantees; - use asap_accuracy::test_support::raw_data_samples_sign_statistic; - let root = asap_types::ir::QueryRoot::Operator(node.clone()); - assert!(bind_guarantees(&root, &Clone::clone).is_none()); - let proven = bind_guarantees(&root, &|guarantee| { - guarantee.bind(&|statistic| { - (statistic == raw_data_samples_sign_statistic("data")).then_some(1.0) - }) - }) - .expect("a declared nonnegative source recovers the ratio contract"); - assert!(proven.spec.error.evaluate().unwrap() <= 0.01, "{proven:?}"); - assert!( - node.guarantee.is_none(), - "conditional sizing must not attach a hypothetical sign proof" - ); - physical_asap_dag(&node); - // Keep the actual signed-sketch counterexample: division guards alone pass - // even though the quantile interpolation does not preserve relative error. - // The sketch reads its minimum and maximum exactly, so the interpolated - // order statistics are interior ones. - let alpha = (0.01 - 8.0 * f64::EPSILON) / 2.01; - let values = [-2., -1., 1.011, 2.]; - let estimate = |q| { - let mut sketch = asap_sketchlib::DdSketch::new(alpha); - for value in values { - sketch.try_add(&value).unwrap(); - } - interpolated_quantile(&sketch, q) - }; - let numerator = estimate(0.5); - let denominator = estimate(0.9); - let got = numerator / denominator; - assert!( - numerator.is_finite() && denominator.is_finite() && denominator != 0. && got.is_normal() - ); - let exact_quantile = |q: f64| { - let rank = q * (values.len() - 1) as f64; - let (lower, upper) = (rank.floor() as usize, rank.ceil() as usize); - values[lower] * (1. - rank.fract()) + values[upper] * rank.fract() - }; - let want = exact_quantile(0.5) / exact_quantile(0.9); - let error = (got - want).abs() / want.abs(); - assert!( - error > 0.01, - "fixture must expose cancellation beyond the requested budget" - ); -} - -/// Interpolated DDS quantiles require a source-domain proof; extrema do not -/// interpolate, and a computed value must not inherit its source's sign. -#[test] -fn ddsketch_interpolation_requires_a_nonnegative_domain() { - use asap_accuracy::propagation::bind_guarantees; - use asap_accuracy::test_support::raw_data_samples_sign_statistic; - use asap_types::ir::{schema::SketchAlgorithm, QueryRoot}; - for (query, endpoint, raw_samples) in [ - ("quantile_over_time(0, data[5m])", true, true), - ("quantile_over_time(1, data[5m])", true, true), - ("quantile_over_time(0.5, data[5m])", false, true), - ("quantile(0.5, data - 100)", false, false), - ] { - let node = explicit_summary_candidate( - lower_promql(query, AccuracyTarget::max_error(0.01)).unwrap(), - SketchAlgorithm::DDSketch, - ); - let bound = node - .guarantee - .as_ref() - .and_then(|g| g.spec.error.evaluate()); - assert_eq!(bound.is_some(), endpoint, "{query}"); - let proven = bind_guarantees(&QueryRoot::Operator(node), &|guarantee| { - guarantee.bind(&|statistic| { - (statistic == raw_data_samples_sign_statistic("data")).then_some(1.0) - }) - }); - assert_eq!( - proven.and_then(|g| g.spec.error.evaluate()).is_some(), - raw_samples, - "{query}" - ); - } -} - -/// Rewriting the inner average (its own target, as sum and count -/// accumulators) must not invalidate an otherwise usable outer sketch. -#[test] -fn quantile_over_temporal_average_keeps_a_legal_candidate() { - for query in [ - "quantile(0.9, avg_over_time(a[5m]))", - "quantile(0.9, avg_over_time(a[5m]) + avg_over_time(b[5m]))", - ] { - let accuracy = AccuracyTarget::max_error(0.01); - let pre = lower_promql(query, accuracy.clone()).unwrap(); - let node = explicit_local_candidates(pre) - .into_iter() - .find(|node| { - OperatorNode::reachable(node).iter().any(|n| { - matches!( - n.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) - }) - }) - .unwrap_or_else(|| panic!("outer sketch candidate must survive: {query}")); - physical_asap_dag(&node); - } -} - -#[test] -fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { - use asap_types::ir::schema::{NonNegativeWeightProof, SketchAlgorithm, WeightDomain}; - for is_count in [true, false] { - let query = if is_count { - "topk(1, count_over_time(up[5m]))" - } else { - "topk(1, sum_over_time(up[5m]))" - }; - let accuracy = AccuracyTarget::max_error(0.01); - let pre = lower_promql(query, accuracy.clone()).unwrap(); - let candidates = explicit_local_candidates(pre); - // The heap state each summarized candidate builds. - let heap = |node: &Rc| { - OperatorNode::reachable(node).into_iter().find(|n| { - matches!( - n.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) - }) - }; - let wanted = if is_count { - SketchAlgorithm::CmsWithHeap - } else { - SketchAlgorithm::CountSketchWithHeap - }; - let (node, state) = candidates - .iter() - .find_map(|node| { - let state = heap(node)?; - let (family, _, _) = aggregate(&state); - matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &wanted) - .then_some((node, state)) - }) - .expect("weighted sketch candidate"); - let (family, update, _) = aggregate(&state); - if is_count { - assert_eq!(update.weight, SummaryInputExpr::Constant(1.)); - assert_eq!( - update.weight_domain, - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount - } - ); - } else { - assert_eq!( - update.weight, - SummaryInputExpr::Column(ColumnRef::SampleValue) - ); - // A signed sum's Count-Min heap carries no non-negative proof, - // so plan selection rejects it. - for state in candidates.iter().filter_map(heap) { - let (family, update, _) = aggregate(&state); - if matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::CmsWithHeap) - { - assert!(!matches!( - update.weight_domain, - WeightDomain::NonNegative { .. } - )); - } - } - } - for value in [1., 0., 3., -3.] { - let key = asap_sketchlib::DataInput::Str("a"); - let mut cms: asap_sketchlib::CountMinF64 = - asap_sketchlib::CountMinSketch::with_dimensions(5, 128); - let mut cs = - asap_sketchlib::CountSketch::>::with_dimensions( - 5, 128, - ); - for _ in 0..10 { - let weight = contribution(family, update, value); - if is_count { - cms.update_weighted(&key, weight); - } else { - cs.update_weighted(&key, weight); - } - } - let got = if is_count { - cms.estimate(&key) - } else { - cs.estimate(&key) - }; - assert_eq!(got, if is_count { 10. } else { 10. * value }); - } - physical_asap_dag(node); - } -} - -// A rejected composition must fail the assertion helper without relying on a planner bug. -#[test] -#[should_panic(expected = "does not compose")] -fn logical_candidates_fails_on_a_candidate_that_does_not_compose() { - asap_integration_tests::planning::require_composed_operator_candidates(vec![ - asap_devtools::planning_stages::ShownLogicalCandidate { - id: "invalid-composition".into(), - roots: Err("invalid choice".into()), - }, - ]); -} diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index a2e6670d3..2f0f0c7e2 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -6,29 +6,24 @@ //! the summary-bound shape node by node, including the family `(Kind, //! Params)` committed on each edge's schema. -use asap_types::stages::HowToComputeAggregateMeasure; use std::rc::Rc; use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::physical_export::{ - at_ingestion, at_ingestion_physical_asap_dag, physical_asap_dag, -}; +use asap_integration_tests::physical_export::physical_asap_dag; use asap_integration_tests::planning::{ logical_candidates, logical_candidates_ignoring_guarantees, }; -use asap_plan_selection::reference_pricer::WholePlanSelection as Selection; -use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::AccuracyMetric; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::operator::arguments::Reduction; + +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::DataType; -use asap_types::ir::schema::{ - EntityIdentity, ExactKind, ExactParams, FieldDataType, GroupingStrategy, Schema, - SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, +use asap_types::ir::schema::{FieldDataType, Schema}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; +use asap_types::summary_type::{ + EntityIdentity, ExactKind, ExactParams, SketchAlgorithm, SketchStatistic, SummaryInputExpr, + SummaryUpdate, }; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::physical::PhysicalWorkloadPlan; /// The first logical candidate (Stage 1, in enumeration order) of `root` as /// a query at `accuracy` that uses an ASAP operator, or the plain exact one @@ -50,9 +45,9 @@ fn realize(root: &Rc, accuracy: AccuracyTarget) -> Rc bool { match &node.operator { Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), + summary_type: FieldDataType::Sketch(kind), .. - }) if kind.algorithm() == algorithm => true, + }) if kind == algorithm => true, _ => node .children() .iter() @@ -254,90 +249,6 @@ fn promql_binary_arithmetic_preserves_nested_structure_and_vector_matching() { } } -#[test] -fn planner_only_e2e_temporal_topk_preserves_query_update_and_evaluation_contract() { - // Self-contained Planner E2E: each case starts from PromQL text and ends - // at the post-ASAP summary DAG. No controller/backend types, - // fixtures, configuration, or runtime are involved. - let cases = [ - ( - "topk by (service) (5, count_over_time(requests[1m]))", - SummaryInputExpr::Constant(1.0), - SketchAlgorithm::CmsWithHeap, - vec![ColumnRef::Named("service".into())], - ), - ( - "topk(5, sum_over_time(requests[1m]))", - SummaryInputExpr::Column(ColumnRef::SampleValue), - SketchAlgorithm::CountSketchWithHeap, - vec![], - ), - ]; - for (source, expected_update, expected_family, excluded_labels) in cases { - let pre = lower_promql( - source, - AccuracyTarget::max_error(0.01).failure_probability(0.01), - ) - .expect("lower temporal Top-K"); - let candidates = heap_topk_candidates_using( - &pre, - AccuracyTarget::max_error(0.01).failure_probability(0.01), - expected_family, - ); - assert!( - !candidates.is_empty(), - "heap-backed temporal Top-K candidate" - ); - // The search archive may visit tighter local error allocations first. - // Every visited configuration must preserve the same update contract. - for candidate in candidates { - let Some(ASAPOp::SummaryEstimate { - summary_input, - query: SketchStatistic::TopK { k, .. }, - }) = candidate.asap() - else { - panic!("expected Top-K estimate, got {:?}", candidate.operator) - }; - assert_eq!( - *k, 5, - "the requested Top-K cardinality must survive binding" - ); - let Some(ASAPOp::SummaryAgg { - input: state_input, - family, - child, - .. - }) = summary_input.asap() - else { - panic!("expected structured Top-K state input") - }; - let FieldDataType::Sketch(kind, _) = family else { - panic!("expected a heap-backed sketch family, got {family:?}") - }; - assert_eq!(kind.algorithm(), &expected_family); - let heap_size = match kind.params() { - SketchParams::CmsWithHeap { heap_size, .. } - | SketchParams::CountSketchWithHeap { heap_size, .. } => *heap_size, - params => panic!("expected heap-bearing Top-K parameters, got {params:?}"), - }; - assert!( - heap_size >= 100u32.max(*k as u32), - "a tighter local error budget may increase, never reduce, capacity" - ); - assert_eq!( - state_input.item.as_ref(), - Some(&SummaryInputExpr::EntityIdentity( - EntityIdentity::PromqlLabelSet { - excluding: excluded_labels.clone() - } - )) - ); - assert_eq!(state_input.weight, expected_update); - assert!(!child.contains_asap()); - } - } -} - /// Execute the ungrouped temporal TopK subset with exact state. This tests /// the emitted update contract, not sketch approximation or backend execution. fn execute_topk_reference(plan: &OperatorNode) -> Vec<(String, f64)> { @@ -471,7 +382,7 @@ fn planner_heap_topk_reference_execution_matches_ground_truth() { /// /// ```text /// SummaryEstimate { query: Quantile{0.99} } → {quantile: Float64} -/// └─ SummaryAgg { Kll{k:269}, input: SampleValue } → {state: Sketch(Kll, {k:269})} +/// └─ SummaryAgg { Kll{k:269}, input: SampleValue } → {state: Sketch(Kll)} /// └─ FinalizeExactAccumulator /// └─ SummaryAgg { Rate, input: SampleValue } → {ts, state: ExactAggregate(Rate), …} /// └─ TimeRange{5m} → Scan → {ts, value} @@ -523,7 +434,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { // same empty `by: []` (issue #163). let Some(ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, reduction, .. @@ -531,19 +442,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { else { panic!("expected SummaryAgg, got {:?}", summary_input.operator); }; - assert_eq!( - family, - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) - ); + assert_eq!(family, &FieldDataType::Sketch(SketchAlgorithm::Kll)); assert_eq!(input, &SummaryUpdate::column(ColumnRef::SampleValue)); assert_eq!( reduction, @@ -552,16 +451,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { ); assert_eq!( dtype(&summary_input.schema, "state"), - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) + &FieldDataType::Sketch(SketchAlgorithm::Kll) ); let Some(ASAPOp::FinalizeExactAccumulator { child }) = child.asap() else { @@ -573,7 +463,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { // grouping concept at all — every entity stays its own summary. let Some(ASAPOp::SummaryAgg { child: leaf, - family, + summary_type: family, reduction, .. }) = child.asap() @@ -622,255 +512,6 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { ); } -#[test] -fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { - // Real workload selection must expose the state-to-value edge; an outer - // sketch must not interpret exact accumulator bytes as input samples. - let pre = lower_promql( - "quantile(0.9, sum_over_time(m[1m]))", - AccuracyTarget::max_error(0.05), - ) - .unwrap(); - // The KLL candidate over the inner exact Sum accumulator. - let plan = candidates_using(&pre, AccuracyTarget::max_error(0.05), SketchAlgorithm::Kll) - .into_iter() - .find(|plan| { - matches!(plan.asap(), Some(ASAPOp::SummaryEstimate { summary_input, .. }) - if matches!(summary_input.asap(), Some(ASAPOp::SummaryAgg { child, .. }) - if matches!(child.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))) - }) - .expect("a KLL over the finalized Sum accumulator"); - // Stored timings are gone: time the plan with its outer summary - // maintained and read the timed copy. - let timed_plan = at_ingestion(&plan); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = timed_plan.asap() else { - panic!("expected selected quantile summary"); - }; - let Some(ASAPOp::SummaryAgg { child, .. }) = summary_input.asap() else { - panic!("expected maintained outer summary"); - }; - let Some(ASAPOp::FinalizeExactAccumulator { child: source }) = child.asap() else { - panic!( - "missing explicit accumulator finalization: {:?}", - child.operator - ); - }; - assert_eq!( - child.timing, - Some(asap_types::ir::properties::ExecutionTiming::IngestionTime) - ); - assert!(matches!( - source.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }) - )); - assert!(child - .schema - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); - assert!(child - .schema - .fields - .iter() - .any(|field| matches!(field.dtype, FieldDataType::Plain(DataType::Float64)))); - // Explicit boundary is a valid post-ASAP DAG. - physical_asap_dag(&plan); -} - -#[test] -fn physical_node_owns_phase_independently_of_binary_payload() { - use asap_types::ir::properties::ExecutionTiming; - // The one-selector case (both operands over the same series, so the - // BinaryOp may run at ingestion) is `arithmetic_over_one_selector_executes` - // in materialization_e2e.rs, ignored until #580. - let query = "sum_over_time(m[1m]) + sum_over_time(n[1m])"; - let expected = ExecutionTiming::QueryTime; - let input = lower_promql(query, AccuracyTarget::max_error(0.05)).unwrap(); - let plan = realize(&input, AccuracyTarget::max_error(0.05)); - let dag = at_ingestion_physical_asap_dag(&plan); - let node = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::BinaryOp { .. }) - ) - }) - .unwrap(); - assert_eq!(node.output_state.timing, expected); - let wire = serde_json::to_value(&node.payload).unwrap(); - assert!(wire.get("timing").is_none()); - let mut obsolete = wire.clone(); - obsolete["timing"] = serde_json::json!(expected.as_str()); - assert!(serde_json::from_value::(obsolete).is_err()); - let restored: PhysicalASAPOperatorPayload = serde_json::from_value(wire).unwrap(); - assert_eq!(restored, node.payload); -} - -/// `query` lowered at `planned` and selected at `accuracy`: the physical -/// candidates whose two summaries are both DDSketch, and plan selection over -/// all of them. -fn ddsketch_ratio_selection( - query: &str, - planned: AccuracyTarget, - accuracy: AccuracyTarget, - non_negative: bool, -) -> (Vec, Selection) { - let root = lower_promql(query, planned).unwrap(); - let mut data = asap_integration_tests::planning::declared_data(&root); - for facts in data.sources.values_mut() { - facts.values_non_negative = non_negative; - } - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let exact = vec![0; inventory.targets.len()]; - let dds = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - *candidate == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch) - }) - .expect("quantile supports DDS") - }) - .collect(); - let entries = [asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: asap_types::workload::QueryRequirements { - accuracy, - max_latency_ms: None, - }, - predictability: Default::default(), - recurrence: Default::default(), - range_evaluation: None, - }]; - // Pass 1 choices composed and implemented by Stage 2; one query, so no - // sharing rule applies. - use asap_accuracy::propagation::recompose_guarantees; - use asap_logical_asap_optimization::test_support::compose_logical_candidate; - let models = asap_planner::PlanningModels::builtin(); - let mut physical = Vec::new(); - for choice in [exact, dds] { - let roots: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let roots = recompose_guarantees(&roots).unwrap(); - physical.extend( - asap_physical_asap_optimization::optimize_physical_asap( - &format!("L{}", physical.len() + 1), - &roots, - &entries, - &data, - models.deployment_capabilities.query_time_results_retained, - ) - .unwrap(), - ); - } - for (index, plan) in physical.iter_mut().enumerate() { - plan.id = format!("fixture-{index}"); - } - let selection = - asap_devtools::memo_pricing::reference_select(&physical, &entries, &data, models).unwrap(); - let ratios = physical - .into_iter() - .filter(|candidate| { - let sketches: Vec<_> = candidate - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(*kind.algorithm()), - _ => None, - }) - .collect(); - sketches == [SketchAlgorithm::DDSketch, SketchAlgorithm::DDSketch] - }) - .collect(); - (ratios, selection) -} - -/// Declared nonnegative inputs permit formal ratio admission; an unknown -/// sign does not. The same explicitly sized DDS fixture is also evaluated -/// numerically on all-positive and all-negative samples without claiming -/// a general bound for arbitrary signed interpolation. -#[test] -fn ddsketch_ratio_checks_sign_contract_and_one_sided_sketch_evaluations() { - for sign in [-1., 1.] { - let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let (ratios, selection) = ddsketch_ratio_selection( - "quantile_over_time(0.9, data[5m]) / quantile_over_time(0.5, data[5m])", - AccuracyTarget::max_error(0.01), - target.clone(), - sign > 0.0, - ); - let accepted: Vec<_> = ratios - .iter() - .filter(|candidate| selection.costs.contains_key(&candidate.id)) - .collect(); - assert!( - !ratios.is_empty(), - "explicit DDS fixture exists regardless of admission" - ); - assert_eq!(!accepted.is_empty(), sign > 0.0, "{selection:?}"); - // Both explicit operands use the planner's composed-budget alpha. - let alphas: Vec = ratios[0] - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => match kind.params() { - SketchParams::DDSketch { alpha } => Some(*alpha), - _ => None, - }, - _ => None, - }) - .collect(); - assert_eq!(alphas.len(), 2); - assert_eq!(alphas[0], alphas[1]); - let alpha = alphas[0]; - let bound = target.max_error; - for values in [ - vec![sign; 30], - (1..=100).map(|i| sign * i as f64).collect(), - vec![sign, sign, sign, sign * 30., sign * 100.], - ] { - let mut sorted = values.clone(); - sorted.sort_by(f64::total_cmp); - let exact = |q: f64| { - let r = q * (sorted.len() - 1) as f64; - sorted[r.floor() as usize] * (1. - r.fract()) - + sorted[r.ceil() as usize] * r.fract() - }; - let mut sketch = asap_sketchlib::DdSketch::new(alpha); - for v in values { - sketch.try_add(&v).unwrap(); - } - // PromQL's quantile: interpolated between neighbouring order - // statistics. - let interpolated = |q: f64| sketch.quantile_interpolated(q).unwrap().unwrap(); - let want = exact(0.9) / exact(0.5); - let got = interpolated(0.9) / interpolated(0.5); - assert!((got - want).abs() / want.abs() <= bound + 1e-12); - } - } -} - // Every `without` aggregation candidate exports a valid DAG: its summary state // column carries the family instead of the evaluation's Float64 value. #[test] diff --git a/crates/integration-tests/tests/range_evaluation.rs b/crates/integration-tests/tests/range_evaluation.rs deleted file mode 100644 index 2f97eeb72..000000000 --- a/crates/integration-tests/tests/range_evaluation.rs +++ /dev/null @@ -1,423 +0,0 @@ -//! #757: a query answered as a range request (`RangeEvaluation`), e.g. a -//! dashboard panel that evaluates the last hour at 1-minute steps on every -//! refresh. Stage 1 and Stage 2 keep tumbling-window widths on the step grid, and -//! Stage 3 prices `span_ms / step_ms + 1` evaluations per request, credits -//! the tumbling windows kept across steps and requests, and retains kept tumbling windows for -//! the span plus the window. - -use crate::planner_layering_common; - -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_plan_selection::RejectionCause; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::ASAPOp; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RangeEvaluation, RepeatedDemand, RepeatingEntry, - RepetitionInterval, SourceFacts, -}; -use planner_layering_common::*; - -/// The dashboard's range: the last hour at 1-minute steps, 61 evaluations -/// per request. -const LAST_HOUR_BY_MINUTE: RangeEvaluation = RangeEvaluation { - step_ms: 60_000, - span_ms: 3_600_000, -}; - -/// Price explicitly named computations under the same models as production. -/// This fixture controls its tumbling-window width; a timed MILP archive is not a catalog -/// of every legal materialization or a stable source of P1/P2 identifiers. -fn run_explicit_for(workload: &PlanningWorkload, deployment: &DeploymentCapabilities) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::ir::schema::SketchAlgorithm; - use asap_types::stages::HowToComputeAggregateMeasure; - let roots = lower_promql(workload); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - assert_eq!(inventory.targets.len(), 1); - let compose = |choice| compose_logical_candidate(&inventory, &[choice]).unwrap(); - let mut candidates = vec![compose(0)]; - if let Some(choice) = inventory.targets[0] - .ways_to_compute - .iter() - .position(|candidate| { - matches!( - candidate, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - { - candidates.push(compose(choice)); - } - let build = candidates.last().unwrap(); - let grids = workload - .query_workload - .entries() - .map(|entry| entry.evaluation_grid_ms()) - .collect::>(); - let tumbling_windows = tumbling_window_candidates(build, &grids); - assert_eq!( - tumbling_windows.len(), - 1, - "one explicit tumbling window computation" - ); - candidates.push( - window_composition_pass2_candidates(build, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(), - ); - let settings = asap_cost::CostInputs::default(); - run_explicit_candidates( - workload, - roots, - candidates, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(deployment), - ) -} - -fn run_explicit(workload: &PlanningWorkload) -> Run { - run_explicit_for(workload, &DeploymentCapabilities::default()) -} - -/// Pattern B (#509 Example 3: `quantile_over_time(0.99, latency_ms[5m])` -/// every minute over 1M series) with each request answered as `range`. -fn pattern_b_with(range: Option) -> PlanningWorkload { - let mut workload = pattern_b(); - for entry in workload - .query_workload - .repeating_queries - .iter_mut() - .flatten() - { - entry.range_evaluation = range; - } - workload -} - -/// A deployment that keeps query-time results across evaluations (B3). -fn keeping_query_time_results() -> DeploymentCapabilities { - DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - } -} - -/// `query` every `interval_ms` over 1M series of `m` sampled every 1.5 s, -/// arriving continuously, exact, answered as `range`. -fn exact_panel(query: &str, interval_ms: u32, range: Option) -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(interval_ms)), - requirements: QueryRequirements::default(), - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: range, - }]), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - sources: [( - "m".to_string(), - SourceFacts { - series_count: Some(1_000_000), - sample_interval: Some(DurationMs(1_500)), - ..SourceFacts::default() - }, - )] - .into(), - }), - } -} - -fn cost(run: &Run, id: &str) -> f64 { - run.cost(id).unwrap_or_else(|| panic!("{id} is not priced")) -} - -fn close(actual: f64, expected: f64) -> bool { - (actual - expected).abs() <= 1e-6 * expected.abs().max(1.0) -} - -/// Each physical candidate's tumbling-window chains: (tumbling-window width, materialization). -fn tumbling_window_chains(run: &Run) -> Vec<(String, u64, Materialization)> { - run.physical - .iter() - .flat_map(|p| { - p.dag().nodes.iter().filter_map(|n| match &n.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some((p.id.clone(), *tumbling_window_width_ms, n.materialization)), - _ => None, - }) - }) - .collect() -} - -/// The candidate whose single tumbling-window chain is materialized as `wanted`. -fn with_chain(run: &Run, wanted: Materialization) -> String { - tumbling_window_chains(run) - .into_iter() - .find(|(_, _, m)| *m == wanted) - .unwrap_or_else(|| panic!("no candidate with a {wanted:?} chain")) - .0 -} - -/// (a) The issue's acceptance example: Pattern B as a dashboard over the -/// last hour at 1-minute steps prices 61 evaluations per request. Every -/// query-time computation runs at every step, so the whole-window KLL -/// (2.3625 → 144.1125 cost/s), the tumbling windows rebuilt per step (3.6112 → -/// 220.2852) and the exact plan (2.4576 → 149.9136) cost 61 times as much, -/// the memory each step holds while it runs included. The kept tumbling windows (B3) -/// are built once per request (one new 1-min tumbling window) but merged at every -/// step, and keep 64 older tumbling windows instead of 4 (1 h + 5 min of tumbling windows): -/// 436.59 → 6,987.05. B1 keeps 66 instead of 6: 654.54 → 7,202.31. The -/// whole-window KLL is still selected. -#[test] -fn stage3_plan_selection_prices_61_evaluations_per_dashboard_request() { - let deployment = keeping_query_time_results(); - let none = run_explicit_for(&pattern_b_with(None), &deployment); - let range = run_explicit_for(&pattern_b_with(Some(LAST_HOUR_BY_MINUTE)), &deployment); - assert_eq!(LAST_HOUR_BY_MINUTE.evaluations_per_request(), 61); - for (id, before, after) in [ - ("P1", 2.457_600_019_2, 149.913_601_171_2), - ("P2", 2.362_500_025_2, 144.112_501_537_2), - ("P3", 3.611_233_350_8, 220.285_234_398_8), - ("P3-m1", 654.542_718_733_8, 7_202.305_718_761_8), - ("P3-m2", 436.587_770_004_24, 6_987.053_970_131_44), - ] { - assert!(close(cost(&none, id), before), "{id}: {}", cost(&none, id)); - assert!(close(cost(&range, id), after), "{id}: {}", cost(&range, id)); - } - // Every query-time computation that is not kept runs once per step. - for id in ["P1", "P2", "P3"] { - assert!(close(cost(&range, id), 61.0 * cost(&none, id)), "{id}"); - } - // B3: 4 → 64 tumbling windows kept between requests; B1: 6 → 66 tumbling windows kept. - let keeps = |run: &Run, id: &str, bytes: &str| { - assert!( - run.selection.costs[id] - .per_node - .values() - .any(|n| n.detail.contains(&format!("; keeps {bytes} bytes"))), - "{id} keeps {bytes} bytes" - ) - }; - keeps(&none, "P3-m2", "3488000000"); - keeps(&range, "P3-m2", "55808000000"); - keeps(&none, "P3-m1", "5232000000"); - keeps(&range, "P3-m1", "57552000000"); - assert_eq!(range.selection.selected, "P2"); -} - -/// (b) For this explicit fixture, refreshing every minute at 30-second steps -/// supplies tumbling windows of -/// `gcd(5 min, 1 min, 30 s) = 30 s`, on the step grid, and Stage 2 keeps -/// them; without the range, the fixture uses 1 min tumbling windows. Both configurations -/// must divide the evaluation step; production also offers finer legal widths. -#[test] -fn explicit_tumbling_window_candidates_follow_the_evaluation_step_grid() { - let step_30s = RangeEvaluation { - step_ms: 30_000, - span_ms: 3_600_000, - }; - let range = run_explicit(&exact_panel("max_over_time(m[5m])", 60_000, Some(step_30s))); - let widths: Vec = tumbling_window_chains(&range) - .into_iter() - .map(|(_, w, _)| w) - .collect(); - assert!(!widths.is_empty(), "tumbling windows are offered"); - assert!(widths.iter().all(|&w| w == 30_000), "{widths:?}"); - assert!(range.pruned.is_empty(), "{:?}", range.pruned); - - let none = run_explicit(&exact_panel("max_over_time(m[5m])", 60_000, None)); - let widths: Vec = tumbling_window_chains(&none) - .into_iter() - .map(|(_, w, _)| w) - .collect(); - assert!( - !widths.is_empty() && widths.iter().all(|&w| w == 60_000), - "{widths:?}" - ); -} - -/// (c) A plan that keeps tumbling windows reports how long the runtime must keep -/// them: the window plus the range-evaluation span, 5 min + 1 h = -/// 3,900,000 ms, so the next refresh's earliest step finds its tumbling windows. -/// Without the range it is the window, 300,000 ms. -#[test] -fn stage3_plan_selection_reports_the_retention_of_kept_tumbling_windows() { - let deployment = keeping_query_time_results(); - let query = "max_over_time(m[5m])"; - let range = run_explicit_for( - &exact_panel(query, 60_000, Some(LAST_HOUR_BY_MINUTE)), - &deployment, - ); - let kept = with_chain( - &range, - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ); - assert_eq!(range.selection.selected, kept); - let retention: Vec = range - .selection - .tumbling_window_retention_ms - .values() - .copied() - .collect(); - assert_eq!(retention, [3_900_000]); - - // Pattern B without a range selects the whole-window KLL, which keeps - // nothing; B1 reports its window when it is the plan (no raw data). - let none = run_explicit(&pattern_b_with(None)); - assert!(none.selection.tumbling_window_retention_ms.is_empty()); - let b1 = run_explicit_for( - &pattern_b_with(None), - &DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }, - ); - let retention: Vec = b1 - .selection - .tumbling_window_retention_ms - .values() - .copied() - .collect(); - assert_eq!(retention, [PATTERN_B_WINDOW_MS]); -} - -/// (d) Without a range evaluation nothing changes: Pattern B's pinned -/// costs (#509 Example 3 and 4: whole-window KLL 2.3625, tumbling windows 3.6112, B3 -/// 436.59, B1 5.232 GB kept) are as before, and a range of one step per -/// request (`span_ms: 0`) prices exactly as no range. -#[test] -fn no_range_evaluation_leaves_every_price_unchanged() { - let deployment = keeping_query_time_results(); - let none = run_explicit_for(&pattern_b_with(None), &deployment); - assert!(close(cost(&none, "P2"), 2.362_500_025_2)); - assert!(close(cost(&none, "P3"), 3.611_233_350_8)); - assert!(close(cost(&none, "P3-m2"), 436.587_770_004_24)); - assert_eq!( - none.selection.costs["P3-m1"].usage.average_memory_bytes, - 5_232_101_733.337_067 - ); - let one_step = run_explicit_for( - &pattern_b_with(Some(RangeEvaluation { - step_ms: 60_000, - span_ms: 0, - })), - &deployment, - ); - assert_eq!(one_step.selection.costs, none.selection.costs); - assert_eq!(one_step.selection.selected, none.selection.selected); -} - -/// The example where declaring the range makes Stage 3 prefer sharing: -/// `max_over_time(m[5m])` every minute over 1M series. As one evaluation -/// per minute, rebuilding the window (P1, 20.8281 cost/s) beats the kept -/// tumbling windows (P2-m2, 32.2630: 224 MB of 56-byte per-series maxima kept) and the -/// tumbling windows maintained at ingestion (P2-m1, 46.2340). As a dashboard over the -/// last hour at 1-minute steps, rebuilding costs 61 times as much -/// (1,270.5151), while the kept tumbling windows add only the per-step merges and 1 h -/// of tumbling windows (460.0410), and B1 472.2760: the kept tumbling windows are selected, and without -/// query-time retention, the ingestion-time tumbling windows. -#[test] -fn stage3_plan_selection_prefers_kept_tumbling_windows_once_the_range_is_declared() { - let query = "max_over_time(m[5m])"; - let deployment = keeping_query_time_results(); - let none = run_explicit_for(&exact_panel(query, 60_000, None), &deployment); - let range = run_explicit_for( - &exact_panel(query, 60_000, Some(LAST_HOUR_BY_MINUTE)), - &deployment, - ); - let kept = with_chain( - &range, - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ); - let ingestion = with_chain(&range, Materialization::MatAtIngestion(Medium::Memory)); - for (run, id, expected) in [ - (&none, "P1", 20.828_117), - (&none, kept.as_str(), 32.262_967), - (&none, ingestion.as_str(), 46.234_035), - (&range, "P1", 1_270.515_127), - (&range, kept.as_str(), 460.040_967), - (&range, ingestion.as_str(), 472.276_035), - ] { - assert!( - (cost(run, id) - expected).abs() < 1e-5, - "{id}: {}", - cost(run, id) - ); - } - assert_eq!(none.selection.selected, "P1"); - assert_eq!(range.selection.selected, kept); - let p1 = range - .selection - .rejected - .iter() - .find(|r| r.id == "P1") - .unwrap(); - assert_eq!(p1.cause, RejectionCause::Costlier); - - let without_retention = run_explicit(&exact_panel(query, 60_000, Some(LAST_HOUR_BY_MINUTE))); - assert_eq!(without_retention.selection.selected, ingestion); -} - -/// (e) A chain over an offset input reaches back its offset further: with -/// no raw data kept, `max_over_time(m[60m] offset 10m)` every 10 minutes -/// keeps six 10-minute tumbling windows ending 10 minutes before each evaluation, so -/// its tumbling windows must be kept for 10 min + 60 min = 4,200,000 ms, and the kept -/// memory Stage 3 prices covers that seventh tumbling window; without the offset the -/// retention is the window, 3,600,000 ms. -#[test] -fn an_offset_chain_is_retained_and_priced_for_its_offset() { - let no_raw_data = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plain = run_explicit_for( - &exact_panel("max_over_time(m[60m])", 600_000, None), - &no_raw_data, - ); - let offset = run_explicit_for( - &exact_panel("max_over_time(m[60m] offset 10m)", 600_000, None), - &no_raw_data, - ); - let retention = |run: &Run| -> Vec { - run.selection - .tumbling_window_retention_ms - .values() - .copied() - .collect() - }; - assert_eq!(retention(&plain), [3_600_000]); - assert_eq!(retention(&offset), [4_200_000]); - let memory = |run: &Run| { - run.selection.costs[&run.selection.selected] - .usage - .average_memory_bytes - }; - assert!( - memory(&offset) > memory(&plain), - "{} vs {}", - memory(&offset), - memory(&plain) - ); -} diff --git a/crates/integration-tests/tests/reuse_rules.rs b/crates/integration-tests/tests/reuse_rules.rs deleted file mode 100644 index 2c8d5a4ad..000000000 --- a/crates/integration-tests/tests/reuse_rules.rs +++ /dev/null @@ -1,589 +0,0 @@ -//! Reuse in Stage 1 (#680): the nested-sum merge (a Pass 1 candidate) is -//! selected over the nested sums and answers the same; the rollup and exact -//! top-k reuse candidates (Pass 2 memo rules) are selected once the finer -//! computation they read is materialized for all its readers, and answer as -//! the computation they replace. Selection prices fixed plans made of the -//! memo's candidates, so no assertion depends on a timed search. -use crate::physical_common; - -use std::collections::{BTreeMap, BTreeSet}; -use std::rc::Rc; - -use asap_devtools::planner_layering_examples::{shared_data_workload, SHARED_ROWS_PER_SECOND}; -use asap_executor::physical_planner::{compile, promql_rows}; -use asap_executor::runtime::Scope; -use asap_executor::values::Value; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{ - cheapest_of, compose_memo_plan, expanded_memo, select_among, -}; -use asap_types::ir::physical_export::compile_physical_asap_dag; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, OperatorNode, QueryRoot, -}; -use asap_types::physical::Materialization; -use asap_types::stages::{root_groups, MemoGroupId}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use asap_types::workload::{ - DataArrival, DataWorkload, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, - RepetitionInterval, -}; - -const EVALUATION_MS: i64 = 300_000; - -/// `queries` lowered at `accuracy`, with series identity, as the planner -/// receives them. -fn roots(queries: &[(&str, AccuracyTarget)]) -> Vec { - queries - .iter() - .map(|(query, accuracy)| { - let root = lower_promql(query, accuracy.clone()).unwrap(); - let root = promql_rows::with_series_identity(&root).unwrap(); - QueryRoot::Operator(root) - }) - .collect() -} - -fn exact_queries<'a>(queries: &[&'a str]) -> Vec<(&'a str, AccuracyTarget)> { - queries - .iter() - .map(|query| (*query, AccuracyTarget::exact())) - .collect() -} - -/// The memo Stage 1 builds for `queries`, each with `recurrence`. -fn memo(queries: &[(&str, AccuracyTarget)], recurrence: QueryRecurrence) -> Memo { - expanded_memo(roots(queries), &vec![exact(recurrence); queries.len()]) -} - -/// The workload entry of an exact query evaluated as `recurrence` says. -fn exact(recurrence: QueryRecurrence) -> QueryWorkloadEntry { - QueryWorkloadEntry { - query: Default::default(), - requirements: Default::default(), - predictability: Default::default(), - recurrence, - range_evaluation: None, - } -} - -/// The #509 shared data workload: 1,000,000 series sampled every 15 s. -fn data() -> DataWorkload { - let mut data = shared_data_workload( - DataArrival::Ingesting { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - "m", - ); - // 100 jobs: each `by (job)` group holds many series, so a ranking - // shared between two top-k saves work. - data.sources.get_mut("m").unwrap().distinct_values_per_label = - [("job".to_string(), 100)].into(); - data -} - -/// Every minute. -fn repeating() -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))) -} - -/// Two jobs of two series each, a sample every 15 s over [0, 5m). -fn samples() -> Vec<(&'static str, &'static str, i64, f64)> { - let mut rows = vec![]; - for (s, (job, series)) in [("a", "a1"), ("a", "a2"), ("b", "b1"), ("b", "b2")] - .into_iter() - .enumerate() - { - for step in 0..20i64 { - rows.push(( - job, - series, - step * 15_000, - (s as i64 * 100 + step) as f64 + 0.5, - )); - } - } - rows -} - -/// Execute `root` at query time over the `samples` bound as raw data at its -/// `Scan`; each `TimeRange` keeps its own window (a tumbling window's moved -/// back by the `TimeShift` below it). -fn execute(root: &Rc) -> Vec { - let timed = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - // Tumbling windows run as their tumbling windows, each over its own window. - let timed = - asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains(&timed) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - let store = physical_common::raw_data_store(&dag, |_, schema| { - samples() - .into_iter() - .map(|(job, series, ts, value)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ("job".to_string(), job.to_string()), - ("series".to_string(), series.to_string()), - ]); - promql_rows::series_row(schema, &labels, ts, value).unwrap() - }) - .collect() - }); - let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - let mut rows: Vec = physical_common::execute( - &plan, - store, - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - ) - .remove(0) - .iter() - .flat_map(|batch| batch.to_rows().to_vec()) - .map(|row: Vec| format!("{row:?}")) - .collect(); - rows.sort(); - rows -} - -fn summary_builds(root: &Rc) -> usize { - OperatorNode::reachable(root) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .count() -} - -/// Every candidate of every group of `memo` with origin in `origins`, with -/// the group it implements; groups that answer a query or aggregate only. -fn candidates_of<'a>( - memo: &'a Memo, - origins: &'a [OriginOfCandidateForMemoGroup], -) -> impl Iterator)> + 'a { - let root_groups = root_groups(&memo.query_roots); - memo.candidates - .iter() - .enumerate() - .filter(move |(group, _)| { - let is_root = root_groups.contains(&MemoGroupId(*group as u32)); - let group = &memo.groups[*group]; - is_root - || matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) - }) - .flat_map(move |(group, candidates)| { - candidates - .iter() - .filter(move |c| origins.contains(&c.origin)) - .map(move |c| (group, &c.root_node)) - }) -} - -/// Plan selection's choice among fixed plans of `memo`'s `queries`: every -/// group on its first candidate (`group`), and `variant{i}` per entry of -/// `variants`, a group's candidate index each. Returns the selected plan's -/// name, its roots, whether it builds some computation once per evaluation -/// for all its readers (`MatAtQuery`), and the selection's lowest prices of -/// `group` and of every variant. -fn select( - memo: &Memo, - queries: usize, - entry: QueryWorkloadEntry, - variants: &[BTreeMap], -) -> (String, Vec>, bool, f64, f64) { - let mut plans = vec![( - "group".to_string(), - compose_memo_plan(memo, queries, &BTreeMap::new()), - )]; - for (i, choice) in variants.iter().enumerate() { - plans.push(( - format!("variant{i}"), - compose_memo_plan(memo, queries, choice), - )); - } - let named: Vec<(&str, Vec)> = plans - .iter() - .map(|(name, roots)| { - ( - name.as_str(), - roots.iter().cloned().map(QueryRoot::Operator).collect(), - ) - }) - .collect(); - let (physical, selection) = select_among(&named, &vec![entry; queries], &data()); - let selected = physical - .iter() - .find(|plan| plan.id == selection.selected) - .expect("the selected plan"); - let name = selected.from_logical.clone(); - let built_once = selected - .materialization - .values() - .any(|m| matches!(m, Materialization::MatAtQuery(_))); - let roots = plans - .iter() - .find(|(n, _)| *n == name) - .map(|(_, roots)| roots.clone()) - .unwrap(); - let variants = (0..variants.len()) - .map(|i| cheapest_of(&selection, &format!("variant{i}"))) - .fold(f64::INFINITY, f64::min); - let own = cheapest_of(&selection, "group"); - // A rejected baseline would make "cheaper than the baseline" hold trivially. - assert!(own.is_finite(), "the plan without reuse is valid"); - (name, roots, built_once, own, variants) -} - -/// `(group, candidate index)` of every candidate [`candidates_of`] yields. -fn choices(memo: &Memo, origins: &[OriginOfCandidateForMemoGroup]) -> Vec<(usize, usize)> { - candidates_of(memo, origins) - .map(|(group, node)| { - let index = memo.candidates[group] - .iter() - .position(|c| Rc::ptr_eq(&c.root_node, node)) - .unwrap(); - (group, index) - }) - .collect() -} - -/// [`candidates_of`], without the candidates that are their group's own -/// node. -fn own_candidates<'a>( - memo: &'a Memo, - origins: &'a [OriginOfCandidateForMemoGroup], -) -> impl Iterator)> + 'a { - candidates_of(memo, origins) - .filter(|(group, node)| !Rc::ptr_eq(node, &memo.groups[*group].logical_node)) -} - -/// `sum by (job) (sum_over_time(m[5m]))`: the nested sums and their merge -/// into one accumulator over the raw data samples. Plan selection picks the -/// merge, which builds no per-series state, and every candidate answers the -/// same per-job sums. -#[test] -fn nested_sum_merge_is_selected_and_answers_as_the_nested_sums() { - // Evaluated once, so no tumbling-window candidates. - let query = "sum by (job) (sum_over_time(m[5m]))"; - let memo = memo(&exact_queries(&[query]), QueryRecurrence::Unknown); - let variants: Vec<_> = choices(&memo, &[OriginOfCandidateForMemoGroup::Pass1]) - .into_iter() - .map(|(group, index)| BTreeMap::from([(group, index)])) - .collect(); - let (_, roots, ..) = select(&memo, 1, exact(QueryRecurrence::Unknown), &variants); - assert_eq!(summary_builds(&roots[0]), 1); - let answer = execute(&roots[0]); - assert_eq!(answer.len(), 2, "one sum per job: {answer:?}"); - every_candidate_answers_alike(query); -} - -/// Kinds of the nodes, other than the source chain, that query 0's and -/// query 1's roots share: what one query reads of the other's computation. -fn shared_kinds(roots: &[Rc]) -> BTreeSet { - let first: BTreeSet<_> = OperatorNode::reachable(&roots[0]) - .iter() - .map(Rc::as_ptr) - .collect(); - OperatorNode::reachable(&roots[1]) - .into_iter() - .filter(|node| first.contains(&Rc::as_ptr(node))) - .map(|node| node.operator.kind_name().to_string()) - .filter(|kind| !["Scan", "TimeRange", "TimeShift", "Project"].contains(&kind.as_str())) - .collect() -} - -/// The nodes the plan selected for `queries` shares between its two -/// queries, among the queries' own roots and the plans reading `origin` -/// candidates (the `i`-th of each group that has some, together). Stage 2 -/// offers each computation two queries read built once per evaluation -/// (`MatAtQuery`); the reuse plan is selected and is cheaper than without it. -fn selected_reuse(queries: &[&str], origin: OriginOfCandidateForMemoGroup) -> BTreeSet { - let memo = memo(&exact_queries(queries), repeating()); - let mut by_group: BTreeMap> = BTreeMap::new(); - for (group, index) in choices(&memo, std::slice::from_ref(&origin)) { - by_group.entry(group).or_default().push(index); - } - let most = by_group.values().map(Vec::len).max().unwrap_or(0); - assert!(most > 0, "{queries:?}: some {origin:?} candidate"); - let variants: Vec> = (0..most) - .map(|i| { - by_group - .iter() - .map(|(group, indices)| (*group, indices[i.min(indices.len() - 1)])) - .collect() - }) - .collect(); - let (name, roots, built_once, own, reuse) = - select(&memo, queries.len(), exact(repeating()), &variants); - assert!(name.starts_with("variant"), "{queries:?}: selected {name}"); - assert!( - built_once, - "{queries:?}: {name} computes its shared state once" - ); - assert!(reuse < own, "{queries:?}: reuse {reuse} vs own {own}"); - shared_kinds(&roots) -} - -/// `sum by (job)` beside `sum by (job, instance)`: with the finer sums -/// computed once, plan selection answers the coarse sum from them. -#[test] -fn rollup_is_selected_when_the_finer_sums_are_computed_once() { - let shared = selected_reuse( - &["sum by (job) (m)", "sum by (job, instance) (m)"], - OriginOfCandidateForMemoGroup::Rollup, - ); - // The coarse sum reads the finer sums' state or finalized values. - assert!( - shared.contains("Aggregate") || shared.contains("SummaryAgg"), - "{shared:?}" - ); -} - -/// Two exact top-k of the per-series rates: with the shared ranking -/// computed once, plan selection reads both from one ranking instead of -/// sorting every series twice. -#[test] -fn exact_topk_reuse_is_selected_when_the_ranking_is_computed_once() { - let shared = selected_reuse( - &[ - "topk by (job) (5, rate(m[1m]))", - "topk by (job) (10, rate(m[1m]))", - ], - OriginOfCandidateForMemoGroup::TopKReuse, - ); - assert!(shared.contains("Sort"), "{shared:?}"); -} - -/// `node` as Stage 2 implements it for `entry` (its first -/// materialization), executed. -fn execute_implemented(node: &Rc, entry: &QueryWorkloadEntry) -> Vec { - let plans = asap_physical_asap_optimization::optimize_physical_asap( - "candidate", - &[QueryRoot::Operator(Rc::clone(node))], - std::slice::from_ref(entry), - &data(), - false, - ) - .expect("Stage 2 implements the candidate"); - match &plans[0].roots[0] { - QueryRoot::Operator(root) => execute(root), - QueryRoot::Scalar(_) => panic!("operator root"), - } -} - -/// Every memo candidate of `query`'s aggregate and root groups other than -/// the group's own node executes to the same rows as that node. -fn every_candidate_answers_alike(query: &str) { - let memo = memo(&exact_queries(&[query]), QueryRecurrence::Unknown); - let entry = exact(QueryRecurrence::Unknown); - let mut checked = 0; - for (group, node) in own_candidates(&memo, &[OriginOfCandidateForMemoGroup::Pass1]) { - assert_eq!( - execute_implemented(node, &entry), - execute_implemented(&memo.groups[group].logical_node, &entry), - "{query}: a candidate of group {group} differs" - ); - checked += 1; - } - assert!( - checked > 0, - "{query}: some candidate other than the group's node" - ); -} - -/// A per-series sum drops the metric name, so grouping by `__name__` over -/// the nested sums groups every series under the empty name; a merge over -/// the raw data samples would group them under `m`. Every candidate answers as -/// the nested sums. -#[test] -fn nested_sum_by_the_metric_name_answers_as_the_nested_sums() { - every_candidate_answers_alike("sum by (__name__) (sum_over_time(m[5m]))"); - every_candidate_answers_alike("sum by (job) (sum_over_time(m[5m]))"); -} - -/// Every rollup or top-k reuse candidate in `memo`, of a group only the -/// queries before `first_approximate_query` read, executes to the rows of -/// the computation it replaces (its group's logical node). -fn reuse_answers_alike_in( - memo: &Memo, - queries: &[(&str, AccuracyTarget)], - first_approximate_query: usize, -) { - let entry = exact(repeating()); - let mut checked = 0; - for (group, node) in candidates_of( - memo, - &[ - OriginOfCandidateForMemoGroup::Rollup, - OriginOfCandidateForMemoGroup::TopKReuse, - ], - ) { - if memo.groups[group] - .reached_by_queries() - .iter() - .any(|&q| q >= first_approximate_query) - { - continue; - } - assert_eq!( - execute_implemented(node, &entry), - execute_implemented(&memo.groups[group].logical_node, &entry), - "{queries:?}: a reuse candidate of group {group} differs" - ); - checked += 1; - } - assert!(checked > 0, "{queries:?}: some reuse candidate"); -} - -/// [`reuse_answers_alike_in`] the memo of the exact `queries`. -fn reuse_answers_alike(queries: &[(&str, AccuracyTarget)]) { - reuse_answers_alike_in(&memo(queries, repeating()), queries, queries.len()); -} - -/// Coarse sums (over merged nested sums), minima and maxima rolled up from -/// the finer ones answer as computed from the raw data samples. -#[test] -fn rollups_answer_as_the_coarse_computations() { - for queries in [ - [ - "sum by (job) (sum_over_time(m[5m]))", - "sum by (job, series) (sum_over_time(m[5m]))", - ], - ["min by (job) (m)", "min by (job, series) (m)"], - ["max by (job) (m)", "max by (job, series) (m)"], - ] { - reuse_answers_alike(&exact_queries(&queries)); - } -} - -/// Top-k read from one larger ranking answer as computed alone. -#[test] -fn exact_topk_reuse_answers_as_each_topk() { - reuse_answers_alike(&exact_queries(&[ - "topk by (job) (1, sum_over_time(m[5m]))", - "topk by (job) (2, sum_over_time(m[5m]))", - ])); -} - -/// Query `q`'s rows when Stage 2 implements `roots` together (its first -/// materialization) for `entries`. -fn execute_plan( - roots: &[Rc], - entries: &[QueryWorkloadEntry], - q: usize, -) -> Vec { - let roots: Vec<_> = roots.iter().cloned().map(QueryRoot::Operator).collect(); - let plans = asap_physical_asap_optimization::optimize_physical_asap( - "plan", - &roots, - entries, - &data(), - false, - ) - .expect("Stage 2 implements the plan"); - match &plans[0].roots[q] { - QueryRoot::Operator(root) => execute(root), - QueryRoot::Scalar(_) => panic!("operator root"), - } -} - -/// A fine and a coarse sum beside two differently grouped quantiles: a plan -/// that shares the quantiles' summary build (summary-capability rule) and -/// rolls the coarse sum up from the fine one (rollup rule) answers each sum -/// as its own computation does (from the raw data samples), and each -/// quantile as the same shared build without the rollup. -#[test] -fn a_rollup_over_a_shared_summary_candidate_answers_as_without_it() { - let quantile = AccuracyTarget::max_error(0.05).failure_probability(0.3); - let queries = [ - ( - "sum by (job) (sum_over_time(m[5m]))", - AccuracyTarget::exact(), - ), - ( - "sum by (job, series) (sum_over_time(m[5m]))", - AccuracyTarget::exact(), - ), - ( - "avg by (job, series) (quantile_over_time(0.99, m[5m]))", - quantile.clone(), - ), - ("avg by (job) (quantile_over_time(0.95, m[5m]))", quantile), - ]; - let memo = memo(&queries, repeating()); - let shared = memo - .groups - .iter() - .find(|group| { - group.reached_by_queries().len() > 1 - && group.reached_by_queries().iter().all(|&q| q >= 2) - && matches!(group.logical_node.asap(), Some(ASAPOp::SummaryAgg { .. })) - }) - .map(|group| Rc::clone(&group.logical_node)) - .expect("the quantiles share a summary build"); - // Each quantile's own candidate that reads the shared build. - let mut sharing: BTreeMap = [2, 3] - .into_iter() - .map(|q| { - memo.candidates - .iter() - .enumerate() - .filter(|(group, _)| memo.groups[*group].reached_by_queries() == [q]) - .flat_map(|(group, candidates)| { - candidates - .iter() - .enumerate() - .map(move |(i, c)| (group, i, c)) - }) - .find(|(_, _, c)| { - OperatorNode::reachable(&c.root_node) - .iter() - .any(|node| Rc::ptr_eq(node, &shared)) - }) - .map(|(group, i, _)| (group, i)) - .unwrap_or_else(|| panic!("Q{} reads the shared build", q + 1)) - }) - .collect(); - let without = compose_memo_plan(&memo, queries.len(), &sharing); - let (group, index) = choices(&memo, &[OriginOfCandidateForMemoGroup::Rollup]) - .into_iter() - .find(|(group, _)| memo.groups[*group].reached_by_queries() == [0]) - .expect("a rollup of the coarse sum"); - sharing.insert(group, index); - let with = compose_memo_plan(&memo, queries.len(), &sharing); - let own = compose_memo_plan(&memo, queries.len(), &BTreeMap::new()); - assert!( - OperatorNode::reachable(&with[2]) - .iter() - .any(|node| OperatorNode::reachable(&with[3]) - .iter() - .any(|other| Rc::ptr_eq(node, other) - && matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. })))), - "both quantiles read one summary build" - ); - let entries = vec![exact(repeating()); queries.len()]; - for q in 0..2 { - assert_eq!( - execute_plan(&with, &entries, q), - execute_plan(&own, &entries, q), - "Q{}", - q + 1 - ); - } - for q in 2..4 { - assert_eq!( - execute_plan(&with, &entries, q), - execute_plan(&without, &entries, q), - "Q{}", - q + 1 - ); - } -} diff --git a/crates/integration-tests/tests/rollup_controlled.rs b/crates/integration-tests/tests/rollup_controlled.rs index d646ded6b..7f05e3911 100644 --- a/crates/integration-tests/tests/rollup_controlled.rs +++ b/crates/integration-tests/tests/rollup_controlled.rs @@ -1,18 +1,13 @@ //! Controlled plans for the viewer's actual distinct-count rollup workload. use asap_cost::{CostInputs, CostModel, ResourceUnitCost}; -use asap_integration_tests::planning::{builds_and_rollups, expanded_memo, select_rollup}; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, rollup_candidates, - rollup_pass2_candidates, -}; + use asap_planner::PlanningModels; -use asap_types::stages::HowToComputeAggregateMeasure; + use asap_types::{ ir::{ - schema::{DataType, Field, Schema, SketchAlgorithm}, + schema::{DataType, Field, Schema}, ASAPOp, OperatorNode, QueryRoot, }, - physical::{Materialization, Medium}, workload::PlanningWorkload, }; use std::{collections::HashSet, rc::Rc}; @@ -46,122 +41,6 @@ fn nodes(roots: &[QueryRoot]) -> Vec> { .collect() } -#[test] -fn controlled_distinct_rollup_prices_the_original_workload() { - let (workload, roots) = fixture(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.as_ref().unwrap(); - let generated = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut settings = CostInputs::default(); - settings.cost_model = CostModel::UsageOverTime(ResourceUnitCost::aws_fargate_us_east_1( - settings.resource_speed, - )); - let models = PlanningModels::builtin().with_cost_inputs(&settings); - let mut accepted = Vec::new(); - for algorithm in [None, Some(SketchAlgorithm::Hll), Some(SketchAlgorithm::Kmv)] { - let choices: Vec<_> = generated - .targets - .iter() - .map(|target| match &algorithm { - None => 0, - Some(a) => target - .ways_to_compute - .iter() - .position( - |c| matches!(c,HowToComputeAggregateMeasure::Summary(found) if found==a), - ) - .expect("family generated"), - }) - .collect(); - let roots: Vec<_> = compose_logical_candidate(&generated, &choices).unwrap(); - // Identical sub-DAGs made one node, as the memo's groups are. - let roots: Vec<_> = asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, Rc::clone(node)), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect(); - let mut variants = vec![("separate", roots.clone())]; - if algorithm.is_some() { - let rules = rollup_candidates(&roots); - assert_eq!(rules.len(), 1, "one service rollup for {algorithm:?}"); - for rolled in rollup_pass2_candidates(&roots, &rules).unwrap() { - variants.push(("rollup", rolled.roots.unwrap())); - } - } - for (kind, roots) in variants { - let family = algorithm - .as_ref() - .map_or("exact".to_string(), |a| format!("{a:?}")); - let id = format!("{family}-{kind}"); - let builds: HashSet<_> = nodes(&roots) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .map(Rc::as_ptr) - .collect(); - let mut plan = - asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - &id, - &roots, - &entries, - data, - models.deployment_capabilities.query_time_results_retained, - &mut |node, _, options| { - let mat = Materialization::MatAtQuery(Medium::Memory); - Some(if builds.contains(&node) && options.contains(&mat) { - mat - } else { - Materialization::NotMaterialized - }) - }, - ) - .unwrap() - .expect("fixed query-time implementation"); - plan.id = id.clone(); - let result = asap_devtools::memo_pricing::reference_select( - &[plan.clone()], - &entries, - data, - models, - ); - match result { - Ok(selection) => { - println!( - "{id}: builds={} cost={:?} accepted={:?}", - builds.len(), - selection.costs[&id].cost_per_second, - selection.accepted - ); - accepted.push(plan); - } - Err(error) => println!("{id}: REJECTED {error}"), - } - for n in nodes(&roots) { - if let Some(ASAPOp::SummaryAgg { family, .. }) = n.asap() { - println!("{id} family={family:?}"); - } - } - } - } - let chosen = - asap_devtools::memo_pricing::reference_select(&accepted, &entries, data, models).unwrap(); - println!("CONTROLLED WINNER {}", chosen.selected); - assert_eq!( - chosen.selected, "Kmv-rollup", - "shared KMV must win the controlled cost comparison" - ); -} - // The production entrypoint must find a sketch rollup under the viewer budget. #[test] fn production_distinct_rollup_finds_shared_kmv() { @@ -180,29 +59,27 @@ fn production_distinct_rollup_finds_shared_kmv() { asap_planner::PlanSearchSettings::default(), ) .unwrap(); - let selection = run + let (plan, selection) = run .selection .as_ref() .expect("validated plan within budget"); - println!( - "production cost={:?} report={:?}", - selection.cost.cost_per_second, selection.search_report - ); let accepted = &selection.accepted_accuracy_per_query; assert_eq!(accepted.len(), 2); assert!(accepted .iter() - .all(|accuracy| !accuracy.accepted_regardless_of_guarantee - && accuracy.sketches.iter().any(|sketch| sketch == "Kmv"))); - assert!(selection.cost.cost_per_second.unwrap() < 2.0e-6); - let plan = &run.selected_physical_plan.as_ref().unwrap().plan; - let builds: HashSet<_> = nodes(&plan.roots) + .all(|accuracy| accuracy.sketches.iter().any(|sketch| sketch == "Kmv"))); + assert!( + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, data, &settings) + < 2.0e-6 + ); + let roots = plan.query_roots().unwrap(); + let builds: HashSet<_> = nodes(&roots) .iter() .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) .map(Rc::as_ptr) .collect(); assert_eq!(builds.len(), 1, "both queries reuse a single state build"); - let rollup = nodes(&plan.roots) + let rollup = nodes(&roots) .iter() .any(|n| matches!(n.asap(), Some(ASAPOp::SummaryMerge { .. }))); assert!( @@ -210,18 +87,3 @@ fn production_distinct_rollup_finds_shared_kmv() { "production search must select the cheaper legal rollup" ); } - -// The production shape without a timed search: among fixed plans of the -// memo (a KMV per query, or both reading one KMV state group), plan -// selection with the built-in models picks one build and one rollup, and -// it costs less than a valid build per query. -#[test] -fn memo_offers_one_kmv_build_and_a_rollup() { - let (workload, roots) = fixture(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.as_ref().unwrap(); - let memo = expanded_memo(roots, &entries); - let (selected, separate, shared) = select_rollup(&memo, 2, &entries, data); - assert!(shared < separate, "{shared} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 1)); -} diff --git a/crates/integration-tests/tests/runtime_handoff.rs b/crates/integration-tests/tests/runtime_handoff.rs deleted file mode 100644 index cadfe8788..000000000 --- a/crates/integration-tests/tests/runtime_handoff.rs +++ /dev/null @@ -1,468 +0,0 @@ -//! What `plan_installable` returns for a deployment runtime (issue #749), -//! checked against the Planner's internals; the runtime's own end-to-end -//! run, with public APIs only, is `crates/planner/tests/runtime_install.rs`. - -use std::collections::BTreeSet; - -use asap_executor::physical_planner::compile; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_planner::{ - e2e_plan, plan_installable, test_support::query_time_reference, FrontendInput, - InstallableDagError, InstallablePlan, InstallablePlanError, PlanError, PlanOutput, - PlanningModels, SelectedPlanReport, UserInput, -}; -use asap_types::ir::physical_export::{ - compile_physical_asap_workload, PhysicalASAPDAG, PhysicalASAPNodeId, -}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{ASAPOp, Operator}; -use asap_types::physical::Materialization; -use asap_types::workload::PlanningWorkload; - -/// #509 Example 4b: one repeating `quantile_over_time` over a 60-minute -/// window, evaluated every 10 minutes, over ingesting data. -fn example4b() -> PlanningWorkload { - serde_json::from_str( - r#"{ - "query_workload": { - "language": "promql", - "repeating_queries": [{ - "query": "quantile_over_time(0.99, latency_ms[60m])", - "demand": {"fixed_interval": 600000}, - "requirements": { - "accuracy": {"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}, - "max_latency_ms": 200.0 - }, - "predictability": {"predictable": {"known_at": null}} - }] - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 1000.0}}, - "sources": {"latency_ms": {"series_count": 1000, "sample_interval": 1000}} - } - }"#, - ) - .expect("Example 4b parses") -} - -/// A deployment that keeps no raw data samples: every summary must be built at -/// ingestion time. -fn without_raw_data() -> DeploymentCapabilities { - DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - } -} - -async fn plan(workload: &PlanningWorkload, capabilities: &DeploymentCapabilities) -> PlanOutput { - let models = PlanningModels::builtin().with_deployment_capabilities(capabilities); - e2e_plan(UserInput::new(workload, FrontendInput::Promql, models)) - .await - .expect("Example 4b plans") -} - -fn materialized_count(dag: &PhysicalASAPDAG) -> usize { - dag.nodes - .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) - .count() -} - -/// Without raw data Stage 3 selects ingestion-time tumbling windows; the facade's -/// export carries that selection rather than resetting every node to -/// query time. -#[tokio::test] -async fn export_carries_the_selected_materialization() { - let capabilities = without_raw_data(); - let output = plan(&example4b(), &capabilities).await; - let dag = output.physical_asap_dag().expect("exports"); - assert!( - materialized_count(&dag) > 0, - "the selected tumbling windows are stored" - ); - assert!(dag - .nodes - .iter() - .any(|n| n.output_state.timing == ExecutionTiming::IngestionTime)); - // Exactly Stage 3's selected plan, as Stage 3 timed it. - assert!(output.selection.is_some(), "Stage 3 selected"); - assert_eq!( - dag, - compile_physical_asap_workload(&output.roots()).expect("the selected plan exports") - ); -} - -/// What a runtime that keeps no raw data samples gets from one facade call: -/// the plan and Stage 3's report of the selection. -async fn installable_with_report( - workload: &PlanningWorkload, -) -> (InstallablePlan, Option) { - let capabilities = without_raw_data(); - let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); - plan_installable(UserInput::new(workload, FrontendInput::Promql, models)) - .await - .expect("Example 4b plans") -} - -async fn installable(workload: &PlanningWorkload) -> InstallablePlan { - installable_with_report(workload).await.0 -} - -fn tumbling_window_chain_count(dag: &PhysicalASAPDAG) -> usize { - dag.nodes - .iter() - .filter(|n| { - matches!( - n.payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .count() -} - -/// The installable plan expands its selected tumbling-window width into contiguous -/// tumbling windows, each a materialized view refreshed at ingestion time, and -/// the executor compiles it. The stage export keeps the chain for costing -/// and display. Its one query is answered by the DAG's one root. -#[tokio::test] -async fn installable_plan_has_its_tumbling_window_chains_expanded() { - let workload = example4b(); - let (installed, report) = installable_with_report(&workload).await; - installed.validate().unwrap(); - let plan = &installed; - let dag = &installed.dag; - let views = plan.materialized_views().unwrap(); - assert_eq!(tumbling_window_chain_count(dag), 0); - assert_eq!(materialized_count(dag), views.len()); - let roots: Vec = dag.roots.iter().map(|&r| r as u64).collect(); - let compiled = compile(dag, &views, &roots).expect("the executor compiles it"); - assert_eq!(plan.query_texts.len(), 1); - assert_eq!( - plan.query_texts[0], - "quantile_over_time(0.99, latency_ms[60m])" - ); - // The query reads exactly the materialized views, and no raw data. - let refreshed: BTreeSet = views.iter().map(|v| v.node_id as u64).collect(); - let slots: BTreeSet = compiled.input_slots().map(|(id, _)| id).collect(); - assert_eq!(slots, refreshed); - assert_tumbling_window_coverage(plan, dag.roots[0], 0, 3_600_000); - assert!( - plan.raw_data_retention().unwrap().is_empty(), - "no raw data kept" - ); - let report = report.as_ref().expect("Stage 3 selected"); - assert!(report.cost_per_second.is_some()); - - let staged = plan_for_stages(&workload).await; - assert_eq!( - tumbling_window_chain_count(&staged), - 1, - "the stage export keeps the chain" - ); -} - -async fn plan_for_stages(workload: &PlanningWorkload) -> PhysicalASAPDAG { - plan(workload, &without_raw_data()) - .await - .physical_asap_dag() - .unwrap() -} - -/// Each summary build of `dag` (`SummaryAgg` or `SummaryTumblingWindowChain`) as its JSON -/// payload, sorted: which families with which parameters it builds. -fn summary_builds(dag: &PhysicalASAPDAG) -> Vec { - let mut builds: Vec = dag - .nodes - .iter() - .filter(|n| { - matches!( - n.payload, - Operator::ASAP( - ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } - ) - ) - }) - .map(|n| serde_json::to_string(&n.payload).unwrap()) - .collect(); - builds.sort(); - builds -} - -/// The correctness reference of the installable plan: the same summaries -/// with the same parameters, every node at query time and nothing materialized, -/// with the same queries bound to its roots. -#[tokio::test] -async fn reference_plan_runs_the_installable_plan_at_query_time() { - let plan = installable(&example4b()).await; - let reference = query_time_reference(&plan).expect("re-times"); - reference.validate().unwrap(); - assert_eq!(materialized_count(&reference.dag), 0); - assert!(reference.materialized_views().unwrap().is_empty()); - // The reference reads the raw data the plan's views were refreshed from. - let [retention] = reference.raw_data_retention().unwrap().try_into().unwrap(); - assert_eq!(retention.lookback_ms(), Some(3_600_000)); - assert!(reference - .dag - .nodes - .iter() - .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); - assert_eq!(summary_builds(&reference.dag), summary_builds(&plan.dag)); - assert_eq!(reference.query_texts, plan.query_texts); -} - -/// A repeating PromQL workload over ingesting `latency_ms`: `queries`, each -/// every 10 minutes at rank error 0.01, δ 0.01. -fn repeating(queries: &[&str]) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|q| { - format!( - r#"{{"query": {q:?}, "demand": {{"fixed_interval": 600000}}, - "requirements": {{"accuracy": {{"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}}, - "max_latency_ms": 200.0}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, - "sources": {{"latency_ms": {{"series_count": 1000, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// p99 and p50 over the same 60 minutes share one chain of tumbling windows: the -/// installable plan materializes one shared set of tumbling windows. -#[tokio::test] -async fn two_queries_on_one_chain_store_its_tumbling_windows_once() { - let plan = installable(&repeating(&[ - "quantile_over_time(0.99, latency_ms[60m])", - "quantile_over_time(0.5, latency_ms[60m])", - ])) - .await; - assert_eq!(plan.query_texts.len(), 2); - let views = plan.materialized_views().unwrap(); - let views_below = |root| { - let below = nodes_below(&plan.dag, root); - views - .iter() - .filter(|view| below.contains(&view.node_id)) - .map(|view| view.node_id) - .collect::>() - }; - assert_eq!( - views_below(plan.dag.roots[0]), - views_below(plan.dag.roots[1]) - ); - assert_eq!(views_below(plan.dag.roots[0]).len(), views.len()); - assert_tumbling_window_coverage(&plan, plan.dag.roots[0], 0, 3_600_000); -} - -/// Planned with raw data kept (everything at query time), so each query's -/// range is a query-time raw-data window. -async fn installable_at_query_time(query: &str) -> Result { - let workload = repeating(&[query]); - plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin(), - )) - .await - .map(|(plan, _)| plan) -} - -/// A raw-data window whose position is not a fixed offset from the -/// evaluation time (a subquery or an offset above the range, an `@` -/// anchor below it), or that ends after the evaluation (a negative offset, -/// which no ingestion run can build in time), is refused with its reason, -/// not passed on with a wrong window or dropped; a plain range is listed. -#[tokio::test] -#[ignore = "fails on main too: the `@` query's selection MILP is infeasible (its scan has no rows_held to price); not a wall-clock issue; #1015"] -async fn raw_data_windows_of_unsupported_shapes_are_refused() { - for (query, reason) in [ - ( - "max_over_time(rate(latency_ms[5m])[1h:1m])", - "a subquery or a TimeShift above the range", - ), - ( - "max_over_time(latency_ms[10m] @ 1000)", - "an @ anchor below the range", - ), - ( - "max_over_time(latency_ms[60m] offset -10m)", - "a negative offset below the range", - ), - ] { - match installable_at_query_time(query).await { - Err(PlanError::Installable(InstallablePlanError::Uninstallable( - InstallableDagError::UnsupportedRawDataWindow { - reason: refused, .. - }, - ))) => assert_eq!(refused, reason, "{query}"), - Err(error) => { - // When planning fails before installation, its no-plan - // diagnostic must keep the concrete refusal reason. - let diagnostic = error.to_string(); - assert!(diagnostic.contains(reason), "{query}: {diagnostic}"); - assert!(diagnostic.contains("raw-data window"), "{diagnostic}"); - } - Ok(plan) => panic!( - "{query}: unexpectedly installed {:?}", - plan.raw_data_retention().unwrap() - ), - } - } - let plain = installable_at_query_time("max_over_time(latency_ms[10m] offset 5m)") - .await - .unwrap(); - // 10 minutes of data ending 5 minutes back reaches 15 minutes back. - let [retention] = plain.raw_data_retention().unwrap().try_into().unwrap(); - assert_eq!(retention.lookback_ms(), Some(900_000)); -} - -/// Example 4b answered as the last hour at 10-minute steps (a range -/// evaluation): each selected tumbling window view is kept for the hour its window reads -/// plus the hour the range spans, 7,200,000 ms, as Stage 3 records for the -/// chain (`Selection::tumbling_window_retention_ms`); without the range, for the hour. -#[tokio::test] -async fn stored_tumbling_windows_carry_stage3_tumbling_window_retention() { - let mut workload = example4b(); - let plain = installable(&workload).await; - assert!(plain - .materialized_views() - .unwrap() - .iter() - .all(|v| v.retention_ms() == 3_600_000)); - let mut json = serde_json::to_value(&workload).unwrap(); - json["query_workload"]["repeating_queries"][0]["range_evaluation"] = - serde_json::json!({"step_ms": 600000, "span_ms": 3600000}); - workload = serde_json::from_value(json).unwrap(); - let ranged = installable(&workload).await; - assert_tumbling_window_coverage(&ranged, ranged.dag.roots[0], 0, 3_600_000); - assert!(ranged - .materialized_views() - .unwrap() - .iter() - .all(|v| v.retention_ms() == 7_200_000)); - let output = plan(&workload, &without_raw_data()).await; - assert_eq!( - output - .tumbling_window_retention_ms - .values() - .copied() - .collect::>(), - [7_200_000] - ); - // A plan file that keeps a tumbling window for less than it reads is refused. - let mut short = ranged.clone(); - let view = ranged.materialized_views().unwrap()[0].node_id; - short - .dag - .nodes - .iter_mut() - .find(|n| n.id == view) - .unwrap() - .retention_ms = Some(0); - assert!(matches!( - short.validate(), - Err(InstallablePlanError::Uninstallable( - InstallableDagError::ViewRetentionTooShort { .. } - )) - )); -} - -/// A chain over an offset input keeps its tumbling windows for its window plus the -/// offset: `[60m] offset 10m` reaches 70 minutes back (4,200,000 ms), the -/// unshifted query's chain 60 minutes (3,600,000 ms). Each materialized view -/// carries its own node's requirement; a runtime that keeps the twins' -/// identical tumbling windows once keeps them for the larger one. -#[tokio::test] -async fn an_offset_chain_keeps_its_tumbling_windows_for_its_window_plus_the_offset() { - let plan = installable(&repeating(&[ - "quantile_over_time(0.99, latency_ms[60m])", - "quantile_over_time(0.99, latency_ms[60m] offset 10m)", - ])) - .await; - assert_tumbling_window_coverage(&plan, plan.dag.roots[0], 0, 3_600_000); - assert_tumbling_window_coverage(&plan, plan.dag.roots[1], 600_000, 4_200_000); - let below: Vec<_> = plan - .dag - .roots - .iter() - .map(|&root| nodes_below(&plan.dag, root)) - .collect(); - for view in &plan.materialized_views().unwrap() { - let required = [3_600_000, 4_200_000] - .into_iter() - .zip(&below) - .filter(|(_, nodes)| nodes.contains(&view.node_id)) - .map(|(retention, _)| retention) - .max() - .expect("materialized tumbling window is consumed"); - assert_eq!(view.retention_ms(), required); - } - plan.validate().unwrap(); -} - -/// Any selected legal tumbling-window width must cover the requested interval exactly. -fn assert_tumbling_window_coverage( - plan: &InstallablePlan, - root: PhysicalASAPNodeId, - start: i64, - end: i64, -) { - let below = nodes_below(&plan.dag, root); - let mut intervals: Vec<_> = plan - .materialized_views() - .unwrap() - .iter() - .filter(|view| below.contains(&view.node_id)) - .map(|view| { - let [window] = view.inputs.raw_data_windows() else { - panic!("tumbling window reads one raw interval: {:?}", view.inputs); - }; - assert!(window.width_ms() > 0 && 600_000 % window.width_ms() == 0); - ( - window.end_offset_ms(), - window.end_offset_ms() + window.width_ms(), - ) - }) - .collect(); - intervals.sort_unstable(); - assert!(!intervals.is_empty()); - assert_eq!(intervals[0].0, start); - assert_eq!(intervals.last().unwrap().1, end); - for pair in intervals.windows(2) { - assert_eq!(pair[0].1, pair[1].0); - } -} - -/// Every node `root` reads, directly or through other nodes. -fn nodes_below(dag: &PhysicalASAPDAG, root: PhysicalASAPNodeId) -> BTreeSet { - let mut below = BTreeSet::new(); - let mut pending = vec![root]; - while let Some(node) = pending.pop() { - for edge in dag.edges.iter().filter(|edge| edge.consumer == node) { - if below.insert(edge.producer) { - pending.push(edge.producer); - } - } - } - below -} - -/// A standalone scalar query (`scalar(sum(latency_ms))`) has no DAG node -/// to answer it through the public API, so it is refused with its entry. -#[tokio::test] -async fn a_scalar_query_is_refused() { - match installable_at_query_time("scalar(sum(latency_ms))").await { - Err(PlanError::Installable(InstallablePlanError::UnsupportedQueryRoot { - entry_index: 0, - })) => {} - other => panic!("{:?}", other.map(|p| p.query_texts)), - } -} diff --git a/crates/integration-tests/tests/scan.rs b/crates/integration-tests/tests/scan.rs index a4227a78c..60894441d 100644 --- a/crates/integration-tests/tests/scan.rs +++ b/crates/integration-tests/tests/scan.rs @@ -11,8 +11,8 @@ use std::time::Duration; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::fixtures::metric_schema; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::Source; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; use asap_types::ir::{ ExprSemantics, NonASAPOp, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, diff --git a/crates/integration-tests/tests/schema.rs b/crates/integration-tests/tests/schema.rs index b496782ea..405b658d6 100644 --- a/crates/integration-tests/tests/schema.rs +++ b/crates/integration-tests/tests/schema.rs @@ -10,7 +10,7 @@ //! are label-preserving and keep the schema open. use asap_integration_tests::fixtures::lower_promql; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; fn lower(q: &str) -> std::rc::Rc { lower_promql(q, AccuracyTarget::exact()) diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/922.json b/crates/integration-tests/tests/sharing_issue_fixtures/922.json index 997650b02..43d8c61e9 100644 --- a/crates/integration-tests/tests/sharing_issue_fixtures/922.json +++ b/crates/integration-tests/tests/sharing_issue_fixtures/922.json @@ -55,7 +55,7 @@ "metrics": { "series_count": 10000, "metric_type": null, - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 10000, "stream_norms": null, "max_distinct_per_evaluation": null, diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/923.json b/crates/integration-tests/tests/sharing_issue_fixtures/923.json index 89cfe117d..753f1fb99 100644 --- a/crates/integration-tests/tests/sharing_issue_fixtures/923.json +++ b/crates/integration-tests/tests/sharing_issue_fixtures/923.json @@ -53,7 +53,7 @@ "m_counter": { "series_count": 1000, "metric_type": "counter", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 10000, "stream_norms": null, "max_distinct_per_evaluation": null, @@ -66,7 +66,7 @@ "m_gauge": { "series_count": 1000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 10000, "stream_norms": null, "max_distinct_per_evaluation": null, diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/924.json b/crates/integration-tests/tests/sharing_issue_fixtures/924.json index 69ed9e5e6..9f955da64 100644 --- a/crates/integration-tests/tests/sharing_issue_fixtures/924.json +++ b/crates/integration-tests/tests/sharing_issue_fixtures/924.json @@ -53,7 +53,7 @@ "m_counter": { "series_count": 10000, "metric_type": "counter", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 15000, "stream_norms": null, "max_distinct_per_evaluation": null, @@ -66,7 +66,7 @@ "m_gauge": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 15000, "stream_norms": null, "max_distinct_per_evaluation": null, diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/925.json b/crates/integration-tests/tests/sharing_issue_fixtures/925.json index d8dd7b0d6..af3001cbe 100644 --- a/crates/integration-tests/tests/sharing_issue_fixtures/925.json +++ b/crates/integration-tests/tests/sharing_issue_fixtures/925.json @@ -53,7 +53,7 @@ "m_counter": { "series_count": 10000, "metric_type": "counter", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 15000, "stream_norms": null, "max_distinct_per_evaluation": null, @@ -66,7 +66,7 @@ "m_gauge": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 15000, "stream_norms": null, "max_distinct_per_evaluation": null, diff --git a/crates/integration-tests/tests/sharing_issue_regressions.rs b/crates/integration-tests/tests/sharing_issue_regressions.rs index 04555299e..8d896009f 100644 --- a/crates/integration-tests/tests/sharing_issue_regressions.rs +++ b/crates/integration-tests/tests/sharing_issue_regressions.rs @@ -2,9 +2,9 @@ use asap_cost::CostInputs; use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; use asap_devtools::planning_stages::run_search_over; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; + use asap_planner::{PlanSearchSettings, PlanningModels}; -use asap_types::stages::HowToComputeAggregateMeasure; + use serde_json::json; use std::path::Path; @@ -48,18 +48,18 @@ fn cost(w: &FuzzWorkload) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); - let selection = run.run.selection.as_ref().unwrap(); - let value = selection.cost.cost_per_second.unwrap(); + let (plan, _) = run.run.selection.as_ref().unwrap(); + let value = + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings); println!("cost={value}"); value } /// #923: deriving the coarse sum costs only a sum over the 150 fine groups. -/// Stage 2 offers that plan; Stage 3 keeps a `MatAtQuery` build whose -/// `NotMaterialized` sibling would avoid a second scan (#1142). +/// Stage 2 offers that plan; Stage 3 selects a `OneEvaluation` build whose +/// not-materialized sibling would avoid a second scan (#1142). #[test] #[ignore = "Stage 3 misses the cheaper plan Stage 2 offers (#1142)"] fn exact_coarse_sum_reuses_fine_sum() { @@ -102,7 +102,7 @@ fn slower_repeat_reuses_faster_sum() { ); } -/// #902: the filtered reader avoids its independent scan and aggregation. +/// #902: the filtered query avoids its independent scan and aggregation. #[test] fn grouped_filtered_quantile_plans() { assert!( @@ -111,363 +111,7 @@ fn grouped_filtered_quantile_plans() { ); } -/// #886: SQL count-ranked TopK offers a heap over raw rows, absorbing the count. -#[test] -fn sql_count_topk_has_a_fused_raw_row_heap() { - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - let roots = Planner::new(FuzzSettings::default()) - .lower(&workload(886)) - .unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let (index, target) = inventory - .targets - .iter() - .enumerate() - .find(|(_, target)| target.fused_computation.is_some()) - .expect("SQL TopK must offer a fused raw-row heap"); - for choice in target.ways_to_compute.len()..target.choice_count() { - let mut selected = vec![0; inventory.targets.len()]; - selected[index] = choice; - let composed = asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &selected, - ) - .unwrap(); - let roots: Vec<_> = composed; - let root = roots[0].as_operator().unwrap(); - for node in asap_types::ir::OperatorNode::reachable(root) { - if let Some(asap_types::ir::ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(_, _), - input, - child, - .. - }) = node.asap() - { - assert_eq!( - input.weight, - asap_types::ir::schema::SummaryInputExpr::Constant(1.0) - ); - assert!(!child.contains_asap(), "heap consumes raw rows"); - } - } - let guarded = Planner::new(FuzzSettings::default()).price(&workload(886), &roots, false); - assert!( - guarded.unwrap_err().contains("NoAccuracyGuarantee"), - "frequency intervals must not be claimed as Top-K membership guarantees" - ); - use asap_executor::values::Value; - let rows = (0..7) - .map(|_| { - vec![ - Value::Timestamp(0), - Value::Utf8("NYC".into()), - Value::Utf8("one-host".into()), - Value::Float64(1.0), - ] - }) - .collect(); - let actual = physical_common::execute_raw_rows_at(root, rows, 1000); - assert_eq!(actual.len(), 1, "{actual:?}"); - assert!( - actual[0] - .iter() - .any(|v| matches!(v, Value::Utf8(s) if s.as_ref() == "one-host")), - "{actual:?}" - ); - assert!( - actual[0] - .iter() - .any(|v| matches!(v, Value::Float64(count) if *count == 7.0)), - "{actual:?}" - ); - let compiled = Planner::new(FuzzSettings { - enforce_accuracy_guarantees: false, - ..Default::default() - }) - .price(&workload(886), &roots, false); - assert!(compiled.is_ok(), "fused heap must compile: {compiled:?}"); - } -} - -/// #923: label projection differences below the same per-series average do not -/// prevent an exact sum from rolling up the finer sum's output. -#[test] -fn exact_rollup_catalog_crosses_identical_per_series_average() { - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(923)) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap(); - let owner = expanded.query_roots[1].as_operator().unwrap().0 as usize; - assert!( - expanded.candidates[owner] - .iter() - .any(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup), - "coarse root has no rollup candidate" - ); -} - -use crate::physical_common; - -/// #902: selecting one group preserves values and SQL's one NULL row when -/// the group is absent, including empty inputs and groups of NULL samples. -#[test] -fn filtered_group_preserves_sql_empty_and_null_semantics() { - use asap_executor::values::Value; - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::ir::{operator::AggIntent, NonASAPOp}; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(902)) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap(); - let (owner, candidate) = expanded.candidates.iter().enumerate().flat_map(|(owner, candidates)| candidates.iter().map(move |c| (owner,c))) - .find(|(_, c)| c.origin == OriginOfCandidateForMemoGroup::Rollup && matches!(c.root_node.non_asap(), Some(NonASAPOp::Aggregate { measures, child, .. }) - if matches!(measures.as_slice(), [AggIntent::Min { .. }]) && matches!(child.non_asap(), Some(NonASAPOp::Filter { .. })))) - .expect("filtered group read candidate"); - // The same readout keeps a KLL's rank bound: it selects a whole group, - // never predicates on an approximate value or combines populations. - use asap_accuracy::propagation::recompose_guarantees; - use asap_logical_asap_optimization::test_support::compose_logical_candidate; - use asap_types::ir::{schema::SketchAlgorithm, QueryRoot}; - use std::rc::Rc; - let filter = candidate.root_node.children()[0]; - let grouped = filter.children()[0]; - let inv = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(Rc::clone(grouped)), - ]), - ) - .unwrap(); - let choices: Vec<_> = inv - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap_or(0) - }) - .collect(); - let approximate = compose_logical_candidate(&inv, &choices).unwrap().remove(0); - let approximate = approximate.as_operator().unwrap(); - let filtered = Rc::new( - filter - .with_new_children(|_| Rc::clone(approximate)) - .unwrap(), - ); - let readout = Rc::new( - candidate - .root_node - .with_new_children(|_| Rc::clone(&filtered)) - .unwrap(), - ); - let recomposed = recompose_guarantees(&[QueryRoot::Operator(readout)]).unwrap(); - assert!(approximate.guarantee.is_some()); - assert_eq!( - recomposed[0].as_operator().unwrap().guarantee, - approximate.guarantee - ); - // A range of keys can select multiple populations; their extrema do not - // inherit one selected quantile's rank bound. - let mut many_groups = (*filtered).clone(); - if let asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { pred, .. }) = - &mut many_groups.operator - { - if let asap_types::ir::ScalarExpr::Compare { op, .. } = &mut pred.0 { - *op = asap_types::ir::scalar::CompareOpKind::Gt; - } - } - let many_groups = Rc::new(many_groups); - let invalid = Rc::new( - candidate - .root_node - .with_new_children(|_| Rc::clone(&many_groups)) - .unwrap(), - ); - assert!( - recompose_guarantees(&[QueryRoot::Operator(invalid)]).unwrap()[0] - .as_operator() - .unwrap() - .guarantee - .is_none() - ); - // A filter that also reads the estimate decides by an approximate value - // which group is kept, so the selected group's bound does not hold. - let mut on_estimate = (*filtered).clone(); - if let asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { pred, .. }) = - &mut on_estimate.operator - { - pred.0 = asap_types::ir::ScalarExpr::BoolAnd(vec![ - pred.0.clone(), - asap_types::ir::ScalarExpr::IsNotNull(Box::new(asap_types::ir::ScalarExpr::Column(1))), - ]); - } - let on_estimate = Rc::new(on_estimate); - let invalid = Rc::new( - candidate - .root_node - .with_new_children(|_| Rc::clone(&on_estimate)) - .unwrap(), - ); - assert!( - recompose_guarantees(&[QueryRoot::Operator(invalid)]).unwrap()[0] - .as_operator() - .unwrap() - .guarantee - .is_none() - ); - let row = |city: &str, value| { - vec![ - Value::Timestamp(0), - Value::Utf8(city.into()), - Value::Utf8("host".into()), - value, - ] - }; - for rows in [ - vec![], - vec![row("LA", Value::Float64(5.0))], - vec![row("NYC", Value::Null)], - vec![ - row("NYC", Value::Float64(10.0)), - row("NYC", Value::Float64(30.0)), - row("LA", Value::Float64(90.0)), - ], - ] { - let before = physical_common::execute_raw_rows_at( - &expanded.groups[owner].logical_node, - rows.clone(), - 1000, - ); - let after = physical_common::execute_raw_rows_at(&candidate.root_node, rows, 1000); - assert_eq!(format!("{after:?}"), format!("{before:?}")); - assert_eq!(after.len(), 1); - } -} - -/// #923: SQL counts roll up by merging integer states, without float casts. -#[test] -fn sql_count_rollup_executes_with_integer_results() { - use asap_executor::values::Value; - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::ir::ASAPOp; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let mut w = workload(902); - w.queries.truncate(2); - w.queries[0].entry.query.0 = - "SELECT city, host, COUNT(*) AS c FROM metrics GROUP BY city, host".into(); - for query in [ - "SELECT city, COUNT(*) AS c FROM metrics GROUP BY city", - "SELECT COUNT(*) AS c FROM metrics", - ] { - w.queries[1].entry.query.0 = query.into(); - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&w) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups( - memo, - ) - .unwrap(); - let (owner, candidate) = expanded - .candidates - .iter() - .enumerate() - .flat_map(|(owner, candidates)| candidates.iter().map(move |c| (owner, c))) - .find(|(_, c)| { - c.origin == OriginOfCandidateForMemoGroup::Rollup - && asap_types::ir::OperatorNode::reachable(&c.root_node) - .iter() - .any(|node| { - matches!(node.asap(), Some(ASAPOp::FinalizeExactAccumulator { child }) - if matches!(child.asap(), Some(ASAPOp::SummaryMerge { group_by:Some(_), .. }))) - }) - }) - .expect("integer count state rollup candidate"); - let rows = vec![ - vec![ - Value::Timestamp(0), - Value::Utf8("NYC".into()), - Value::Utf8("a".into()), - Value::Null, - ], - vec![ - Value::Timestamp(0), - Value::Utf8("NYC".into()), - Value::Utf8("b".into()), - Value::Float64(10.0), - ], - vec![ - Value::Timestamp(0), - Value::Null, - Value::Utf8("a".into()), - Value::Null, - ], - ]; - let before = - physical_common::execute_raw_rows(&expanded.groups[owner].logical_node, rows.clone()); - let after = physical_common::execute_raw_rows(&candidate.root_node, rows); - let sorted = |rows: Vec>| { - let mut rows: Vec<_> = rows.iter().map(|row| format!("{row:?}")).collect(); - rows.sort(); - rows - }; - assert!(after - .iter() - .all(|row| matches!(row.last(), Some(Value::Int64(_))))); - assert_eq!(sorted(before), sorted(after)); - let before_empty = - physical_common::execute_raw_rows(&expanded.groups[owner].logical_node, vec![]); - let after_empty = physical_common::execute_raw_rows(&candidate.root_node, vec![]); - assert_eq!(sorted(before_empty), sorted(after_empty)); - } -} - -/// #922: duplicate exact TopK readers share their result and inner window work. +/// #922: duplicate exact TopK queries share their result and inner window work. #[test] fn duplicate_exact_topk_shares_work() { let mut w = workload(924); @@ -481,7 +125,7 @@ fn duplicate_exact_topk_shares_work() { assert!(duplicate <= single * 1.001, "{duplicate} vs {single}"); } -/// #924/#925 SQL variants: nested windows and slower readers reuse one sum. +/// #924/#925 SQL variants: nested windows and slower queries reuse one sum. #[test] fn sql_sum_shares_nested_windows_and_slower_readers() { let mut w = workload(922); @@ -505,68 +149,3 @@ fn sql_sum_shares_nested_windows_and_slower_readers() { "{repeated_cost} vs {single}" ); } - -/// #923: sketches over an inner range function expose the same finer rollup. -#[test] -fn sketch_rollup_crosses_inner_range_function() { - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let mut w = workload(923); - for (query, keys) in w.queries.iter_mut().zip(["job, region", "job"]) { - query.entry.query.0 = format!("quantile by ({keys}) (0.95, sum_over_time(m_gauge[5m]))"); - query.entry.requirements.accuracy = - asap_types::ir::properties::AccuracyTarget::max_error(0.05).failure_probability(0.01); - } - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&w) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap(); - let owner = expanded.query_roots[1].as_operator().unwrap().0 as usize; - assert!(expanded.candidates[owner] - .iter() - .any(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup)); -} - -/// #886: unsupported per-group SQL count sketches are explicitly pruned; -/// the exact counter and separately typed raw-row TopK fusion remain available. -#[test] -fn sql_grouped_count_sketches_are_not_offered() { - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - use asap_types::ir::{operator::AggIntent, NonASAPOp}; - use asap_types::stages::PruneReason; - let w = workload(886); - let roots = Planner::new(FuzzSettings::default()).lower(&w).unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let (index, target) = inventory.targets.iter().enumerate().find(|(_, target)| matches!(inventory.aggregate(target).non_asap(), Some(NonASAPOp::Aggregate { measures, .. }) if matches!(measures.as_slice(), [AggIntent::Count { .. }]))).unwrap(); - assert!(target - .ways_to_compute - .iter() - .all(|candidate| !matches!(candidate, HowToComputeAggregateMeasure::Summary(_)))); - assert_eq!( - inventory - .pruned_of_target(index, false) - .iter() - .filter(|(_, reason)| *reason == PruneReason::NoRelationalCountSketchRealization) - .count(), - 2 - ); - assert!(inventory - .targets - .iter() - .any(|target| target.fused_computation.is_some())); -} diff --git a/crates/integration-tests/tests/sketch_rollup.rs b/crates/integration-tests/tests/sketch_rollup.rs deleted file mode 100644 index a5044bd34..000000000 --- a/crates/integration-tests/tests/sketch_rollup.rs +++ /dev/null @@ -1,399 +0,0 @@ -//! A regrouping `SummaryMerge` (rollup) answers a coarse grouping from the -//! states of a finer one: `GROUP BY job` from states built `GROUP BY region, -//! job`, merging each job's per-region states. For every mergeable sketch -//! the planner offers a SQL grouped query, the rolled-up answer equals the -//! direct build's (deterministic merges) or is within the sketch's bound -//! (KLL), and its coverage is the direct build's. -use crate::physical_common; - -use std::collections::BTreeMap; -use std::rc::Rc; -use std::sync::Arc; - -use asap_executor::{ - physical_planner::compile, - runtime::{Limits, RunContext, Scope}, - sources::InMemoryDataStore, - values::{Batch, Value}, -}; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::planning::explicit_local_candidates; -use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::summary_coverage::CoverageError; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; -use asap_types::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, SketchAlgorithm, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use futures::StreamExt; - -const REGIONS: [&str; 3] = ["us", "eu", "ap"]; -const JOBS: [&str; 4] = ["api", "web", "db", "cache"]; - -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "http", - Schema::new(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("job", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - Field::plain("user_id", DataType::Float64, false), - ]), - ) -} - -/// 6,000 rows over every (region, job): a heavy-tailed latency and a user -/// id from 800, from a fixed linear congruential generator. -fn rows() -> Vec<(&'static str, &'static str, f64, f64)> { - let mut state: u64 = 0x2545_f491_4f6c_dd1d; - let mut next = move || { - state = state - .wrapping_mul(6_364_136_223_846_793_005) - .wrapping_add(1_442_695_040_888_963_407); - (state >> 33) as f64 / (1u64 << 31) as f64 - }; - (0..6_000) - .map(|_| { - let region = REGIONS[(next() * 3.0) as usize % 3]; - let job = JOBS[(next() * 4.0) as usize % 4]; - let latency = 10.0 / (1.0 - next()).max(1e-6).sqrt(); - let user = (next() * 800.0).floor(); - (region, job, latency, user) - }) - .collect() -} - -/// `node` with every Int64 column typed Float64. The Planner types a SQL -/// `COUNT(DISTINCT)` estimate Int64 and the executor returns Float64, a -/// known SQL execution gap this test does not cover. -fn as_float64(node: &Rc) -> Rc { - let rebuilt = node.with_new_children(as_float64).unwrap(); - let mut schema = node.schema.clone(); - for field in &mut schema.fields { - if field.dtype == FieldDataType::Plain(DataType::Int64) { - field.dtype = FieldDataType::Plain(DataType::Float64); - } - } - Rc::new( - OperatorNode::with_schema(rebuilt.operator, schema).with_guarantee(node.guarantee.clone()), - ) -} - -/// The answers of the logical DAG `root` over [`rows`], by group key. -fn execute(root: &Rc) -> BTreeMap { - let root = &as_float64(root); - let dag = physical_common::compile_physical_asap_dag(root).unwrap(); - let scan = dag - .nodes - .iter() - .find(|node| { - matches!( - &node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { .. }) - ) - }) - .expect("one scan"); - let PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { source, .. }) = &scan.payload else { - unreachable!() - }; - let schema = Arc::new(scan.output_schema.clone()); - let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - let table: Vec> = rows() - .into_iter() - .map(|(region, job, latency, user)| { - vec![ - Value::Timestamp(0), - Value::Utf8(region.into()), - Value::Utf8(job.into()), - Value::Float64(latency), - Value::Float64(user), - ] - }) - .collect(); - let store = InMemoryDataStore::default() - .with_raw_data( - source.clone(), - schema.clone(), - vec![Batch::try_new(schema.clone(), table).unwrap()], - ) - .unwrap(); - let bound = plan.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: 0, - }, - // The UnivMon states planned here are large: the merge's copy of - // twelve of them exceeds the default limit. - Limits { - max_bytes: 1 << 32, - ..Limits::default() - }, - ) - .unwrap(); - futures::executor::block_on(async { - let mut stream = bound.execute(plan.roots(), context).unwrap().remove(0); - let mut answers = BTreeMap::new(); - while let Some(batch) = stream.next().await { - for row in batch.unwrap().to_rows() { - let (keys, value) = row.split_at(row.len() - 1); - let key = keys - .iter() - .map(|value| match value { - Value::Utf8(text) => text.to_string(), - other => format!("{other:?}"), - }) - .collect::>() - .join(","); - let value = match value[0] { - Value::Float64(value) => value, - Value::Int64(value) => value as f64, - ref other => panic!("not a number: {other:?}"), - }; - assert!(answers.insert(key, value).is_none(), "one row per group"); - } - } - answers - }) -} - -/// The only `SummaryAgg` of `root`. -fn build(root: &Rc) -> Option> { - let builds: Vec<_> = OperatorNode::reachable(root) - .into_iter() - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect(); - match builds.as_slice() { - [build] => Some(Rc::clone(build)), - _ => None, - } -} - -/// `node` with `from` replaced by `to`. -fn replace( - node: &Rc, - from: &Rc, - to: &Rc, -) -> Rc { - if Rc::ptr_eq(node, from) { - return Rc::clone(to); - } - Rc::new( - node.with_new_children(|child| replace(child, from, to)) - .unwrap() - .with_guarantee(node.guarantee.clone()), - ) -} - -/// `coarse`, a build by `job`, as the same build by `(region, job)` -/// regrouped by `job`: each output state merges its job's per-region -/// states. -fn rolled_up(coarse: &OperatorNode) -> Rc { - let Some(ASAPOp::SummaryAgg { - child, - family, - input, - reduction: Reduction::Reduce(by), - grouping, - filter, - }) = coarse.asap() - else { - panic!("a grouped build: {:?}", coarse.operator) - }; - let region = child.schema.column_id("region").unwrap(); - assert_eq!(by.keys().len(), 1); - let finer = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: family.clone(), - input: input.clone(), - reduction: Reduction::by(vec![region, by.keys()[0]]), - grouping: grouping.clone(), - filter: filter.clone(), - })) - .unwrap(); - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![finer], - group_by: Some(vec![1]), - newest_tumbling_windows: None, - })) - .unwrap() -} - -/// Every local sketch rule for `sql` (one build by `job`), each with its -/// rollup from `(region, job)`: the family, the direct root and the -/// rolled-up root. Explicit rule fixtures keep this execution test independent -/// of the search archive and its tighter, potentially much larger capacities. -async fn direct_and_rolled_up( - sql: &str, - accuracy: AccuracyTarget, -) -> Vec<(SketchAlgorithm, Rc, Rc)> { - let root = lower_sql(sql, &catalog(), accuracy.clone()).await.unwrap(); - let mut out = Vec::new(); - for candidate in explicit_local_candidates(root) { - let Some(build) = build(&candidate) else { - continue; - }; - let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = build.asap() - else { - continue; - }; - let merge = rolled_up(&build); - assert_eq!( - asap_types::test_support::definition_id(&merge.coverage().expect("a valid rollup")), - asap_types::test_support::definition_id(&build.coverage().unwrap()), - "{kind:?}: the rolled-up states cover what the direct build does" - ); - let rolled = replace(&candidate, &build, &merge); - rolled.validate_structure().unwrap(); - out.push((*kind.algorithm(), candidate, rolled)); - } - out -} - -/// Distinct users per job: KMV, HLL and UnivMon states of `(region, job)` -/// merged per job equal each job's state built directly (KMV keeps the k -/// smallest hashes of the union, HLL each register's maximum, UnivMon's -/// counters add), so every answer is the direct build's. -#[tokio::test] -async fn distinct_counts_rolled_up_equal_the_direct_build() { - let accuracy = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_metric(AccuracyMetric::Relative); - let pairs = direct_and_rolled_up( - "SELECT job, COUNT(DISTINCT user_id) AS value FROM http GROUP BY job", - accuracy, - ) - .await; - let families: Vec<_> = pairs.iter().map(|(family, ..)| *family).collect(); - assert!(families.contains(&SketchAlgorithm::Kmv), "{families:?}"); - for (family, direct, rolled) in pairs { - let (direct, rolled) = (execute(&direct), execute(&rolled)); - assert_eq!(direct.len(), JOBS.len()); - assert_eq!(rolled, direct, "{family:?}"); - } -} - -/// p99 latency per job. DDSketch's bucket counts add, so the merged state -/// is the direct build's and so is its answer. KLL's compactions are -/// randomized, so its rolled-up answer differs, but stays within its rank -/// bound of the exact p99, as the direct build's does. -#[tokio::test] -async fn quantiles_rolled_up_keep_the_sketch_bound() { - const EPSILON: f64 = 0.05; - for metric in [AccuracyMetric::Rank, AccuracyMetric::Relative] { - let accuracy = AccuracyTarget::max_error(EPSILON) - .failure_probability(0.01) - .accuracy_metric(metric); - let pairs = direct_and_rolled_up( - "SELECT job, approx_percentile_cont(latency, 0.99) AS value FROM http GROUP BY job", - accuracy, - ) - .await; - for (family, direct, rolled) in pairs { - let (direct, rolled) = (execute(&direct), execute(&rolled)); - assert_eq!(direct.len(), JOBS.len()); - match family { - SketchAlgorithm::DDSketch => assert_eq!(rolled, direct), - SketchAlgorithm::Kll => { - for (job, answer) in rolled { - let mut latencies: Vec = rows() - .into_iter() - .filter(|row| row.1 == job) - .map(|row| row.2) - .collect(); - latencies.sort_by(f64::total_cmp); - let rank = latencies.partition_point(|&v| v <= answer) as f64 - / latencies.len() as f64; - assert!( - (rank - 0.99).abs() <= EPSILON, - "{job}: rank {rank} of {answer}" - ); - } - } - other => panic!("unexpected quantile family {other:?}"), - } - } - } -} - -/// A regrouping merge keys only by group keys of its input states (or by -/// none, a global aggregate), and never merges counter states of different -/// groups. -#[tokio::test] -async fn invalid_rollups_are_rejected() { - let accuracy = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_metric(AccuracyMetric::Relative); - let root = lower_sql( - "SELECT job, COUNT(DISTINCT user_id) AS value FROM http GROUP BY job", - &catalog(), - accuracy.clone(), - ) - .await - .unwrap(); - let candidate = explicit_local_candidates(root) - .into_iter() - .find(|candidate| { - build(candidate).is_some_and(|build| { - matches!( - build.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) - }) - }) - .unwrap(); - let merge = rolled_up(&build(&candidate).unwrap()); - let finer = Rc::clone(merge.children()[0]); - let regroup = |keys: Vec| { - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&finer)], - group_by: Some(keys), - newest_tumbling_windows: None, - })) - .map_err(|error| error.to_string()) - }; - // The state column is no key; nor is a column twice. - for keys in [vec![2], vec![1, 1]] { - assert!(regroup(keys.clone()).is_err(), "{keys:?}"); - } - // No keys: the global build over the same rows. - assert!(regroup(vec![]).unwrap().coverage().is_some()); - // Both keys, swapped, are still group keys of the input. - assert!(regroup(vec![1, 0]).unwrap().coverage().is_some()); - // Rate states merge consecutive windows of one series, not groups. - let Some(ASAPOp::SummaryAgg { - child, - input, - reduction, - grouping, - .. - }) = finer.asap() - else { - unreachable!() - }; - let rates = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: None, - })) - .unwrap(); - let merged_rates = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![rates], - group_by: Some(vec![1]), - newest_tumbling_windows: None, - })); - assert!(merged_rates - .unwrap_err() - .to_string() - .contains(&CoverageError::FamilyCannotRegroup.to_string())); -} diff --git a/crates/integration-tests/tests/sketch_rollup_planning.rs b/crates/integration-tests/tests/sketch_rollup_planning.rs deleted file mode 100644 index 2519479b7..000000000 --- a/crates/integration-tests/tests/sketch_rollup_planning.rs +++ /dev/null @@ -1,176 +0,0 @@ -//! Sketch rollup in planning: the memo's rollup rule offers answering a -//! coarse sketch grouping from the finest build of the same sketch over the -//! same rows, and plan selection selects it when one build and a merge cost -//! less than a build per query over the raw rows. Selection prices fixed -//! plans made of the memo's candidates, so no assertion depends on a timed -//! search. -use asap_executor::physical_planner::promql_rows; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{builds_and_rollups, expanded_memo, select_rollup}; -use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::QueryRoot; -use asap_types::workload::{ - DataWorkload, Predictability, QueryRecurrence, QueryRequirements, QueryWorkloadEntry, - RepeatedDemand, RepetitionInterval, -}; - -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "http", - Schema::new(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("user_id", DataType::Float64, false), - Field::plain("latency", DataType::Float64, false), - ]), - ) -} - -/// A predictable query repeating every `interval_ms` at `accuracy`. -fn entry(accuracy: &AccuracyTarget, interval_ms: u32) -> QueryWorkloadEntry { - QueryWorkloadEntry { - query: Default::default(), - requirements: QueryRequirements { - accuracy: accuracy.clone(), - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( - interval_ms, - ))), - range_evaluation: None, - } -} - -/// `http` arrives at 2e6 rows/s (W-RollupHeavy's table), 10,000 series, -/// a million distinct users at most per evaluation, and holds a minute of -/// rows (the catalog has no time index, so a query's time predicate is a -/// filter over every row held). -fn data() -> DataWorkload { - serde_json::from_value(serde_json::json!({ - "arrival": {"ingesting": {"rows_per_second": 2_000_000.0}}, - "sources": {"http": { - "series_count": 10_000, - "sample_interval": 5, - "values_non_negative": true, - "max_distinct_per_evaluation": 1_000_000, - "rows_held": 120_000_000, - }}, - })) - .unwrap() -} - -/// The SQL `queries` lowered at `accuracy`. -async fn sql_roots(queries: &[&str], accuracy: &AccuracyTarget) -> Vec { - let mut roots = Vec::new(); - for query in queries { - let root = lower_sql(query, &catalog(), accuracy.clone()) - .await - .unwrap(); - roots.push(QueryRoot::Operator(root)); - } - roots -} - -/// Distinct users by (region, service) and by service, both at relative -/// ε = 0.05 per value over the same window (W-RollupHeavy's targets): the -/// memo offers the coarse KMV rolled up from the finer one, and plan -/// selection selects one build and a rollup, cheaper than a build per -/// query from the raw rows. -#[tokio::test] -async fn stage3_selects_a_distinct_count_rollup_when_it_is_cheaper() { - let accuracy = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_metric(AccuracyMetric::Relative) - .accuracy_scope(AccuracyScope::PerValue); - let window = "WHERE ts >= now() - INTERVAL '5 minutes'"; - let roots = sql_roots( - &[ - &format!( - "SELECT region, service, COUNT(DISTINCT user_id) AS value FROM http {window} \ - GROUP BY region, service" - ), - &format!( - "SELECT service, COUNT(DISTINCT user_id) AS value FROM http {window} \ - GROUP BY service" - ), - ], - &accuracy, - ) - .await; - let entries = vec![entry(&accuracy, 60_000); 2]; - let memo = expanded_memo(roots, &entries); - let (selected, separate, rolled_up) = select_rollup(&memo, 2, &entries, &data()); - assert!(rolled_up < separate, "{rolled_up} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 1)); -} - -/// p99 latency by (region, service), by region and by service at rank -/// ε = 0.05 (template 13's shape): both coarse groupings roll up from one -/// KLL, and plan selection selects that plan. -#[tokio::test] -async fn coarse_quantile_groupings_roll_up_from_one_kll() { - let accuracy = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(AccuracyScope::PerValue); - let window = "WHERE ts >= now() - INTERVAL '5 minutes'"; - let query = |keys: &str| { - format!( - "SELECT {keys}, approx_percentile_cont(latency, 0.99) AS value FROM http {window} \ - GROUP BY {keys}" - ) - }; - let roots = sql_roots( - &[ - &query("region, service"), - &query("region"), - &query("service"), - ], - &accuracy, - ) - .await; - let entries = vec![entry(&accuracy, 60_000); 3]; - let memo = expanded_memo(roots, &entries); - let (selected, separate, rolled_up) = select_rollup(&memo, 3, &entries, &data()); - assert!(rolled_up < separate, "{rolled_up} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 2)); -} - -/// #840's `quantile by (host, job)` and `quantile by (job)` over 10,000 -/// series, every 10 s at rank ε = 0.01, δ = 0.001: the memo offers the -/// coarse KLL rolled up from the finer one, and its guarantee meets the -/// target, so plan selection selects it. -#[test] -fn promql_quantile_by_job_rolls_up_from_by_host_job() { - let accuracy = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank); - let roots: Vec = [ - "quantile by (host, job) (0.5, data)", - "quantile by (job) (0.5, data)", - ] - .iter() - .map(|query| { - let root = lower_promql(query, accuracy.clone()).unwrap(); - QueryRoot::Operator(promql_rows::with_series_identity(&root).unwrap()) - }) - .collect(); - let data: DataWorkload = serde_json::from_value(serde_json::json!({ - "arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10_000, "metric_type": "gauge", - "sample_interval": 15_000, - // 15 minutes of raw rows. - "rows_held": 600_000}}, - })) - .unwrap(); - let entry = entry(&accuracy, 10_000); - let entries = [entry.clone(), entry]; - let memo = expanded_memo(roots, &entries); - let (selected, separate, rolled_up) = select_rollup(&memo, 2, &entries, &data); - assert!(rolled_up < separate, "{rolled_up} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 1)); -} diff --git a/crates/integration-tests/tests/sql_cardinality.rs b/crates/integration-tests/tests/sql_cardinality.rs index 0a418d782..c0a03c4b8 100644 --- a/crates/integration-tests/tests/sql_cardinality.rs +++ b/crates/integration-tests/tests/sql_cardinality.rs @@ -2,7 +2,6 @@ use crate::physical_common; use asap_executor::values::Value; use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; // COUNT(DISTINCT src_ip) skips NULL and keeps the WHERE, Utf8 identities and @@ -19,7 +18,6 @@ async fn sql_distinct_executes_through_raw_scan_and_native_binding() { let root = lower_sql( "SELECT COUNT(DISTINCT src_ip) AS sources FROM flows WHERE keep", &catalog, - AccuracyTarget::exact(), ) .await .unwrap(); diff --git a/crates/integration-tests/tests/sql_frequency.rs b/crates/integration-tests/tests/sql_frequency.rs index 7bf842055..a560eb274 100644 --- a/crates/integration-tests/tests/sql_frequency.rs +++ b/crates/integration-tests/tests/sql_frequency.rs @@ -4,7 +4,6 @@ use crate::physical_common; use asap_executor::values::Value; use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; const ENTROPY: &str = "SELECT -SUM(p * LN(p)) FROM (\ @@ -27,9 +26,7 @@ fn rows(sources: &[&str]) -> Vec> { } async fn run(sql: &str, sources: &[&str]) -> Option { - let root = lower_sql(sql, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let root = lower_sql(sql, &catalog()).await.unwrap(); let result = physical_common::execute_raw_rows(&root, rows(sources)); match result.as_slice() { [row] => match row.as_slice() { diff --git a/crates/integration-tests/tests/sql_group_bound.rs b/crates/integration-tests/tests/sql_group_bound.rs deleted file mode 100644 index e8497c4fa..000000000 --- a/crates/integration-tests/tests/sql_group_bound.rs +++ /dev/null @@ -1,244 +0,0 @@ -//! A whole-result accuracy target over SQL `GROUP BY` groups (#863): the -//! distinct values the data workload declares for the grouping columns -//! (`SourceFacts::distinct_values_per_label`) bound the groups one -//! evaluation returns, so a per-group sketch can be sized to meet the target -//! over all of them. Without declarations the groups stay unbound and only -//! the exact plan meets it. - -use crate::planner_layering_common; - -use asap_devtools::planning_stages::lower_sql_roots; -use asap_devtools::SqlCatalog; -use asap_types::ir::schema::{DataType, Field, Schema, SketchAlgorithm}; -use asap_types::workload::PlanningWorkload; -use planner_layering_common::*; - -/// The `http` table of the RollupHeavy workload. -fn http_catalog() -> SqlCatalog { - let utf8 = |name| Field::plain(name, DataType::Utf8, false); - let float = |name| Field::plain(name, DataType::Float64, false); - SqlCatalog::new().with_table( - "http", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - utf8("region"), - utf8("service"), - utf8("endpoint"), - utf8("status"), - float("user_id"), - float("latency"), - ], - 0, - vec![], - ), - ) -} - -/// `query` every minute with a whole-result target (`metric` error 0.05, -/// failure probability 0.05) over `http`: 10,000 label combinations, and -/// `declared` as each column's distinct values. -fn workload(query: &str, metric: &str, declared: serde_json::Value) -> PlanningWorkload { - let http = serde_json::json!({ - "series_count": 10000, "sample_interval": 5, "values_non_negative": true, - "max_distinct_per_evaluation": 1000000, "distinct_values_per_label": declared, - "rows_held": 1000000000 - }); - serde_json::from_value(serde_json::json!({ - "query_workload": {"language": {"sql": "datafusion_sql"}, "query_batch": null, - "repeating_queries": [{ - "query": query, "demand": {"fixed_interval": 60000}, - "requirements": {"accuracy": {"accuracy_metric": metric, "max_error": 0.05, - "failure_probability": 0.05, "accuracy_scope": "whole_result"}}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 2e6}}, - "sources": {"http": http}} - })) - .expect("workload") -} - -/// Every `http` column's distinct values: 4 regions, 25 services, 25 -/// endpoints, 4 statuses. -fn every_column() -> serde_json::Value { - serde_json::json!({"region": 4, "service": 25, "endpoint": 25, "status": 4}) -} - -/// Evaluate the exact computation and each supported local summary choice. -/// These tests exercise group/error models, not a timed search's visited archive. -fn run_group_choices(workload: &PlanningWorkload) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let roots = lower_sql_roots(workload, &http_catalog()).expect("lowers"); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - assert_eq!(local.targets.len(), 1, "one grouped statistic per fixture"); - let candidates = (0..local.targets[0].ways_to_compute.len()) - .map(|choice| compose_logical_candidate(&local, &[choice]).unwrap()) - .collect(); - run_explicit_candidates( - workload, - roots, - candidates, - asap_planner::PlanningModels::builtin(), - ) -} - -/// The sketches the selected plan builds (empty for the exact plan). -fn selected_sketches( - query: &str, - metric: &str, - declared: serde_json::Value, -) -> Vec { - let workload = workload(query, metric, declared); - let run = run_group_choices(&workload); - sketch_builds(run.physical(&run.selection.selected).dag()) - .into_iter() - .map(|(_, algorithm, _)| algorithm) - .collect() -} - -/// Template 14: distinct users per (service, endpoint). -const DISTINCT_USERS: &str = "SELECT service, endpoint, COUNT(DISTINCT user_id) AS value \ - FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service, endpoint"; - -/// Template 13: the p99 latency per (region, service). -const P99_LATENCY: &str = "SELECT region, service, approx_percentile_cont(latency, 0.99) AS value \ - FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY region, service"; - -/// With 25 services and 25 endpoints declared, at most 625 groups: a -/// distinct-count sketch per group, each failing with probability at most -/// 0.05 / 625, meets the whole-result target. -#[test] -fn declared_grouping_columns_bound_the_groups_of_a_distinct_count() { - let sketches = selected_sketches(DISTINCT_USERS, "relative", every_column()); - assert!( - matches!( - sketches.as_slice(), - [SketchAlgorithm::Hll | SketchAlgorithm::Kmv] - ), - "{sketches:?}" - ); -} - -/// With 4 regions and 25 services declared, at most 100 groups: a KLL per -/// group meets the whole-result rank target. -#[test] -fn declared_grouping_columns_bound_the_groups_of_a_quantile() { - let sketches = selected_sketches(P99_LATENCY, "rank", every_column()); - assert_eq!(sketches, [SketchAlgorithm::Kll]); -} - -/// Without declared distinct values the groups are unbound: no sketch -/// meets a whole-result target, and the exact plan is selected. -#[test] -fn undeclared_grouping_columns_leave_only_the_exact_plan() { - for (query, metric) in [(DISTINCT_USERS, "relative"), (P99_LATENCY, "rank")] { - let sketches = selected_sketches(query, metric, serde_json::json!({})); - assert!(sketches.is_empty(), "{query}: {sketches:?}"); - } -} - -/// One grouping column declared is not enough: `service` has no declared -/// count, so the groups of `GROUP BY region, service` stay unbound. -#[test] -fn a_partly_declared_grouping_leaves_only_the_exact_plan() { - let sketches = selected_sketches(P99_LATENCY, "rank", serde_json::json!({"region": 4})); - assert!(sketches.is_empty(), "{sketches:?}"); -} - -/// A computed key named like a declared column no longer holds that -/// column's values: `upper(service) AS service` gives no bound. -#[test] -fn a_computed_grouping_column_leaves_only_the_exact_plan() { - let query = "SELECT service, COUNT(DISTINCT user_id) AS value FROM \ - (SELECT concat(service, endpoint) AS service, user_id, ts FROM http) \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service"; - let sketches = selected_sketches(query, "relative", every_column()); - assert!(sketches.is_empty(), "{sketches:?}"); -} - -/// The groups every priced plan's grouped builds and aggregates are priced -/// at (`... into N states`, `... into N groups`), and the rows its scans read. -fn priced_groups_and_scan_rows( - workload: PlanningWorkload, -) -> ( - std::collections::BTreeSet, - std::collections::BTreeSet, -) { - let run = run_group_choices(&workload); - let (mut groups, mut scanned) = ( - std::collections::BTreeSet::new(), - std::collections::BTreeSet::new(), - ); - for cost in run.selection.costs.values() { - for node in cost.per_node.values() { - let words: Vec<&str> = node.detail.split([' ', ':', ';']).collect(); - for window in words.windows(3) { - if window[0] == "into" && matches!(window[2], "states" | "groups") { - groups.insert(window[1].parse().expect("a group count")); - } - if window[0] == "scan" && window[2] == "samples" { - scanned.insert(window[1].parse().expect("a row count")); - } - } - } - } - (groups, scanned) -} - -/// A SQL grouping with an undeclared column gets no invented count: each -/// of the table's 10,000 declared groups of rows is in one group, so its -/// builds are priced at 10,000; without `series_count`, each row read is -/// in at most one group, so they are priced at the rows the scan reads. -/// Declared, `GROUP BY service, endpoint` is 25 x 25. -#[test] -fn an_undeclared_sql_grouping_is_priced_at_the_groups_of_rows_or_the_rows_read() { - let undeclared = || workload(DISTINCT_USERS, "relative", serde_json::json!({})); - let (groups, _) = priced_groups_and_scan_rows(undeclared()); - assert_eq!(groups, std::collections::BTreeSet::from([10_000])); - let mut without_series = undeclared(); - let http = without_series - .data_workload - .as_mut() - .unwrap() - .sources - .get_mut("http") - .unwrap(); - http.series_count = None; - http.sample_interval = None; - let (groups, scanned) = priced_groups_and_scan_rows(without_series); - assert_eq!(groups, scanned); - let (groups, _) = - priced_groups_and_scan_rows(workload(DISTINCT_USERS, "relative", every_column())); - assert_eq!(groups, std::collections::BTreeSet::from([625])); -} - -/// Each endpoint is in one service: declaring `(service, endpoint)`'s 625 -/// combinations prices `GROUP BY service, endpoint` at 625 groups, where -/// the per-column product (25 x 625) is capped only by the 10,000 groups of -/// rows. -#[test] -fn a_declared_column_set_prices_a_sql_grouping_at_its_combinations() { - let hierarchical = - serde_json::json!({"region": 4, "service": 25, "endpoint": 625, "status": 4}); - let (groups, _) = - priced_groups_and_scan_rows(workload(DISTINCT_USERS, "relative", hierarchical.clone())); - assert_eq!(groups, std::collections::BTreeSet::from([10_000])); - let mut declared = workload(DISTINCT_USERS, "relative", hierarchical); - declared - .data_workload - .as_mut() - .unwrap() - .sources - .get_mut("http") - .unwrap() - .distinct_values_per_label_set = vec![asap_types::workload::LabelSetDistinctValues { - labels: vec!["service".to_string(), "endpoint".to_string()], - distinct_values: 625, - }]; - let (groups, _) = priced_groups_and_scan_rows(declared); - assert_eq!(groups, std::collections::BTreeSet::from([625])); -} diff --git a/crates/integration-tests/tests/sql_to_physical.rs b/crates/integration-tests/tests/sql_to_physical.rs deleted file mode 100644 index 258821b74..000000000 --- a/crates/integration-tests/tests/sql_to_physical.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! SQL frontend, the planner pipeline's selection, physical compilation and -//! fresh-run execution. -use crate::physical_common; -use asap_executor::{ - physical_planner::compile, - runtime::{Limits, RunContext, Scope}, - sources::InMemoryDataStore, - values::{Batch, Value}, -}; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::planning::selected_dag; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use futures::StreamExt; -use physical_common::compile_physical_asap_dag; -use std::sync::Arc; - -/// SQL filtering and grouped aggregation survive logical/physical lowering; -/// rebinding the compiled DAG runs against new data rather than cached results. -#[tokio::test] -async fn sql_filter_grouped_sum_executes_and_rebinds() { - let catalog = SqlCatalog::new().with_table( - "metrics", - Schema::new(vec![ - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, true), - ]), - ); - for query in [ - "SELECT service, SUM(value) AS total FROM metrics WHERE value > 1 GROUP BY service", - "SELECT service, SUM(value) AS total FROM metrics GROUP BY service", - ] { - let logical = lower_sql(query, &catalog, AccuracyTarget::exact()) - .await - .unwrap(); - let selected = selected_dag(logical, AccuracyTarget::exact()); - let dag = compile_physical_asap_dag(&selected).unwrap(); - let scan = dag - .nodes - .iter() - .find(|node| { - matches!( - &node.payload, - PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::Scan { .. }) - ) - }) - .expect("raw SQL scan"); - let schema = Arc::new(scan.output_schema.clone()); - assert!(schema - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); - let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - for multiplier in [1., 2.] { - let rows = [ - ("api", Some(2.)), - ("api", Some(3.)), - ("api", None), - ("batch", Some(4.)), - ("batch", Some(1.)), - ] - .into_iter() - .map(|(service, value)| { - schema - .fields - .iter() - .map(|field| match field.name.as_str() { - "service" => Value::Utf8(service.into()), - "value" => { - value.map_or(Value::Null, |value| Value::Float64(value * multiplier)) - } - _ => panic!("unexpected field {field:?}"), - }) - .collect() - }) - .collect(); - let PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::Scan { - source, - predicates: _, - schema: _scan_schema, - }) = &scan.payload - else { - unreachable!() - }; - let store = InMemoryDataStore::default() - .with_raw_data( - source.clone(), - schema.clone(), - vec![Batch::try_new(schema.clone(), rows).unwrap()], - ) - .unwrap(); - let bound = plan.instantiate(Arc::new(store)).unwrap(); - let mut stream = bound - .execute( - plan.roots(), - RunContext::new( - Scope::Query { - evaluation_time_ms: 300_000, - }, - Limits::default(), - ) - .unwrap(), - ) - .unwrap() - .remove(0); - let mut batches = Vec::new(); - while let Some(batch) = stream.next().await { - batches.push(batch.unwrap()); - } - let mut actual: Vec<_> = batches - .iter() - .flat_map(|batch| batch.to_rows()) - .map(|row| { - let Value::Utf8(service) = &row[0] else { - panic!("missing service") - }; - let Value::Float64(value) = row[1] else { - panic!("missing sum") - }; - (service.to_string(), value) - }) - .collect(); - actual.sort_by(|a, b| a.0.cmp(&b.0)); - assert_eq!( - actual, - vec![ - ("api".into(), 5. * multiplier), - ( - "batch".into(), - 4. * multiplier - + if query.contains("WHERE") && multiplier == 1. { - 0. - } else { - multiplier - } - ) - ] - ); - } - } -} diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index 5f894c98e..48e6fb721 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -19,29 +19,13 @@ use std::rc::Rc; -use asap_frontend_sql::{lower_sql, lower_sql_dialect, SqlCatalog}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::physical_export::physical_asap_dag; -use asap_integration_tests::planning::{ - logical_candidates, logical_candidates_ignoring_guarantees, -}; -use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::physical_export::{EdgeRole, PhysicalASAPNodeId, PhysicalASAPOperatorPayload}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::expr_ir::ColumnRef; +use asap_integration_tests::planning::logical_candidates_ignoring_guarantees; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{ - ExactKind, ExactParams, FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, SummaryUpdate, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; -use asap_types::workload::SqlDialect; - -/// The first logical candidate (Stage 1, in enumeration order) of `root` as -/// a query at `accuracy` that uses an ASAP operator, or the plain exact one -/// when none does: the summary-bound form the pins below inspect. -fn realize(root: &Rc, accuracy: AccuracyTarget) -> Rc { - first_summary_bound(logical_candidates(Rc::clone(root), accuracy)) -} +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate}; +use asap_types::plan::physical_asap_dag::{PhysicalASAPNodeId, PhysicalASAPOperatorPayload}; /// [`realize`] with every output accepted whatever its guarantee: a value /// filter over an estimate has no guarantee, so Pass 1 prunes the summary @@ -69,30 +53,6 @@ fn first_summary_bound(candidates: Vec>) -> Rc { node } -/// `node` with every `Project` above it skipped: Stage 1 restores a -/// rewritten aggregate's output names with a `Project`, and SQL wraps every -/// query in one. -fn below_projects(mut node: &Rc) -> &Rc { - while let Some(NonASAPOp::Project { child, .. }) = node.non_asap() { - node = child; - } - node -} - -/// The first `SummaryAgg` reached from `node` through unary nodes, estimates -/// and finalized accumulators. -fn first_summary_agg(node: &Rc) -> &Rc { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { .. }) => node, - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - first_summary_agg(summary_input) - } - _ => first_summary_agg( - unary_child(node).unwrap_or_else(|| panic!("no SummaryAgg below {:?}", node.operator)), - ), - } -} - /// The single input of a unary non-ASAP node (Project, Filter, Sort, ...) or /// of a `FinalizeExactAccumulator`; `None` for anything else. fn unary_child(node: &OperatorNode) -> Option<&Rc> { @@ -140,280 +100,12 @@ fn catalog() -> SqlCatalog { ) } -async fn lower(sql: &str, accuracy: AccuracyTarget) -> Rc { - lower_sql(sql, &catalog(), accuracy) +async fn lower(sql: &str) -> Rc { + lower_sql(sql, &catalog()) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) } -#[tokio::test] -async fn clickhouse_temporal_sql_reuses_rate_and_increase_physical_summaries() { - for (function, expected) in [ - ( - "asap_rate", - FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), - ), - ( - "asap_increase", - FieldDataType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - ), - ] { - let sql = format!( - "SELECT service, {function}(latency, ts, 300000) AS v \ - FROM metrics WHERE bytes > 0 GROUP BY service" - ); - let pre_asap = lower_sql_dialect( - &sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect("explicit temporal SQL must lower"); - let planned = realize(&pre_asap, AccuracyTarget::exact()); - let physical = first_summary_agg(&planned); - let Operator::ASAP(ASAPOp::SummaryAgg { - family, - reduction, - child, - .. - }) = &physical.operator - else { - panic!("expected a shared SummaryAgg, got {:?}", physical.operator); - }; - assert_eq!(family, &expected); - assert_eq!(reduction, &Reduction::PerEntity); - assert!( - is_kept_non_asap(child), - "expected a retained temporal SQL input, got {:?}", - child.operator - ); - assert!( - matches!(child.non_asap(), Some(NonASAPOp::TimeRange { range, child, .. }) - if *range == std::time::Duration::from_secs(300) - && matches!(child.non_asap(), Some(NonASAPOp::Project { .. }))) - ); - } -} - -#[tokio::test] -async fn clickhouse_outer_sum_recursively_binds_inner_temporal_aggregate() { - for (function, window_ms) in [ - ("asap_rate", 300_000), - ("asap_rate", 3_600_000), - ("asap_increase", 300_000), - ] { - let sql = format!( - "SELECT sum(v) AS value FROM (\ - SELECT service, {function}(latency, ts, {window_ms}) AS v \ - FROM metrics GROUP BY service)" - ); - let pre_asap = lower_sql_dialect( - &sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect("nested temporal SQL must lower"); - let root = realize(&pre_asap, AccuracyTarget::exact()); - - fn has_temporal_summary(node: &OperatorNode) -> bool { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate | ExactKind::Increase, _), - .. - }) => true, - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - has_temporal_summary(summary_input) - } - Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => has_temporal_summary(child), - _ => unary_child(node).is_some_and(|child| has_temporal_summary(child)), - } - } - assert!( - has_temporal_summary(&root), - "inner {function} was hidden: {root:?}" - ); - // The outer sum is an exact Sum state over the finalized per-series - // values, so the export keeps both summaries. - let Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) = &first_summary_agg(&root).operator - else { - unreachable!() - }; - assert_eq!( - family, - &FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - let dag = physical_asap_dag(&root); - assert_eq!( - dag.nodes - .iter() - .filter(|node| matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { .. }) - )) - .count(), - 2 - ); - } -} - -#[tokio::test] -async fn sql_full_query_retains_project_and_binds_inner_aggregate() { - let pre_asap = lower( - "SELECT approx_percentile_cont(latency, 0.99) AS p99 FROM metrics", - AccuracyTarget::max_error(0.01), - ) - .await; - let Some(NonASAPOp::Project { - cols: expected_cols, - qualifier: expected_qualifier, - .. - }) = pre_asap.non_asap() - else { - panic!("sanity: a SQL root is a Project, unlike lower_promql's bare Aggregate"); - }; - let root = realize(&pre_asap, AccuracyTarget::max_error(0.01)); - let Some(NonASAPOp::Project { - child, - cols, - qualifier, - }) = root.non_asap() - else { - panic!("expected retained Project root, got {:?}", root.operator); - }; - assert_eq!(cols, expected_cols, "projection expressions and aliases"); - assert_eq!(qualifier, expected_qualifier, "projection qualifier"); - assert_eq!(root.schema.fields[0].name, "p99", "project output schema"); - assert_eq!( - root.schema.fields[0].dtype, - FieldDataType::Plain(DataType::Float64) - ); - assert!( - matches!( - below_projects(child).operator, - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - ), - "the Aggregate under Project must be summary-bound" - ); -} - -/// A relational join remains a read-time node while both derived-table -/// aggregates are independently selected as physical summaries. -#[tokio::test] -async fn sql_join_recursively_binds_both_temporal_aggregate_children() { - let pre_asap = lower_sql_dialect( - "SELECT a.service, a.v / b.v AS ratio FROM \ - (SELECT service, asap_rate(latency, ts, 300000) AS v FROM metrics WHERE service='errors' GROUP BY service) a \ - INNER JOIN \ - (SELECT service, asap_rate(latency, ts, 300000) AS v FROM metrics WHERE service='requests' GROUP BY service) b \ - ON b.service=a.service", - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect("two-subquery rate ratio must lower"); - let root = realize(&pre_asap, AccuracyTarget::exact()); - let Some(NonASAPOp::Project { - child: join, cols, .. - }) = root.non_asap() - else { - panic!( - "expected Project above relational join, got {:?}", - root.operator - ); - }; - assert!(matches!( - &cols[1].expr, - ScalarExpr::Arithmetic { - op: asap_types::ir::scalar::ArithmeticOpKind::Div, - .. - } - )); - let Some(NonASAPOp::Join { - left, - right, - kind, - pred, - }) = join.non_asap() - else { - panic!( - "expected read-time relational join, got {:?}", - join.operator - ); - }; - assert_eq!(kind, &asap_types::ir::operator::JoinKind::Inner); - assert!(matches!( - &pred.0, - ScalarExpr::Compare { - left, - op: asap_types::ir::scalar::CompareOpKind::Eq, - right, - .. - } if matches!( - (left.as_ref(), right.as_ref()), - (ScalarExpr::Column(0), ScalarExpr::Column(2)) - | (ScalarExpr::Column(2), ScalarExpr::Column(0)) - ) - )); - assert_eq!( - join.schema - .fields - .iter() - .map(|field| field.name.as_str()) - .collect::>(), - vec!["service", "v", "service", "v"] - ); - for child in [left, right] { - let Some(NonASAPOp::Project { - child: aggregate, .. - }) = child.non_asap() - else { - panic!( - "derived table Project was not retained: {:?}", - child.operator - ); - }; - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: aggregate }) = - &aggregate.operator - else { - panic!("derived table Project must consume finalized exact values"); - }; - assert!(matches!( - aggregate.operator, - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), - .. - }) - )); - } - assert!(join - .guarantee - .as_ref() - .is_some_and(|value| value.is_exact())); - let dag = physical_asap_dag(&root); - let join_id = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Join { .. }) - ) - }) - .expect("relational join node") - .id; - let roles = dag - .edges - .iter() - .filter(|edge| edge.consumer == join_id) - .map(|edge| edge.role) - .collect::>(); - assert_eq!(roles, vec![EdgeRole::Left, EdgeRole::Right]); -} - #[tokio::test] async fn sql_relational_parents_retain_summary_bound_aggregate() { let pre_asap = lower( @@ -421,7 +113,6 @@ async fn sql_relational_parents_retain_summary_bound_aggregate() { (SELECT service, approx_percentile_cont(latency, 0.9) AS p \ FROM metrics GROUP BY service) t \ WHERE t.p > 100 ORDER BY t.p DESC LIMIT 5", - AccuracyTarget::max_error(0.01), ) .await; let root = realize_ignoring_guarantees(&pre_asap, AccuracyTarget::max_error(0.01)); @@ -469,7 +160,6 @@ async fn sql_filter_keeps_read_predicate_and_summary_population_selection() { (SELECT service, approx_percentile_cont(latency, 0.9) AS p \ FROM metrics WHERE service = 'api' GROUP BY service) t \ WHERE t.p > 100", - AccuracyTarget::max_error(0.01), ) .await; let expected_read_predicate = { @@ -548,196 +238,3 @@ async fn sql_filter_keeps_read_predicate_and_summary_population_selection() { PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Filter { pred, .. }) if *pred == expected_wire ))); } - -#[tokio::test] -async fn sql_quantile_binds_kll_sketch_over_named_column() { - let pre_asap = lower( - "SELECT approx_percentile_cont(latency, 0.99) FROM metrics", - AccuracyTarget::max_error(0.01), - ) - .await; - let planned = asap_integration_tests::planning::explicit_summary_candidate( - pre_asap, - SketchAlgorithm::Kll, - ); - let root = below_projects(&planned); - - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) = &root.operator - else { - panic!("expected SummaryEstimate root, got {:?}", root.operator); - }; - assert!(matches!(query, SketchStatistic::Quantile { q } if *q == 0.99)); - assert_eq!( - root.schema.fields.len(), - 1, - "no GROUP BY — a single output column" - ); - assert_eq!( - root.schema.fields[0].dtype, - FieldDataType::Plain(DataType::Float64), - "the summary-state type must not propagate past the estimate" - ); - - let Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - .. - }) = &summary_input.operator - else { - panic!("expected SummaryAgg, got {:?}", summary_input.operator); - }; - assert_eq!( - family, - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) - ); - assert_eq!( - input, - &SummaryUpdate::column(ColumnRef::Qualified { - table: "metrics".into(), - name: "latency".into(), - }), - "SQL binds the intent's own named input column, not a synthetic sample value" - ); - assert_eq!( - reduction, - &Reduction::by(vec![]), - "global quantile — no GROUP BY, full reduction" - ); - assert_eq!( - summary_input.schema.fields[0].dtype, - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) - ); - - assert!( - is_kept_non_asap(child), - "expected a kept non-ASAP leaf, got {:?}", - child.operator - ); - assert!(matches!(child.non_asap(), Some(NonASAPOp::Scan { .. }))); - assert!( - child - .schema - .fields - .iter() - .all(|f| matches!(f.dtype, FieldDataType::Plain(_))), - "logical edges carry only plain columns" - ); -} - -/// `SELECT COUNT(DISTINCT service) FROM metrics` at ε = 0.01 lowers to -/// `AggIntent::Cardinality` (`sql_lowering.rs::count_distinct_is_cardinality`) -/// — this fixture explicitly selects the HLL rule, independently of the -/// optimizer's cost preference. HLL is sized from ε and the default δ = 0.01 (#694): `m ≥ -/// 1/(2ε²δ)` reaches precision 18, the largest. -#[tokio::test] -async fn sql_count_distinct_with_epsilon_binds_hll_over_named_column() { - let pre_asap = lower( - "SELECT COUNT(DISTINCT service) FROM metrics", - AccuracyTarget::max_error(0.01), - ) - .await; - let planned = asap_integration_tests::planning::explicit_summary_candidate( - pre_asap, - SketchAlgorithm::Hll, - ); - let root = below_projects(&planned); - - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) = &root.operator - else { - panic!("expected SummaryEstimate root, got {:?}", root.operator); - }; - assert!(matches!(query, SketchStatistic::Cardinality)); - assert_eq!( - root.schema.fields[0].dtype, - FieldDataType::Plain(DataType::Int64), - "COUNT(DISTINCT …) reads back out as an integer count" - ); - - let Operator::ASAP(ASAPOp::SummaryAgg { - family, - input, - reduction, - .. - }) = &summary_input.operator - else { - panic!("expected SummaryAgg, got {:?}", summary_input.operator); - }; - assert_eq!( - family, - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 18, - relative_error: Some(0.01) - } - ), - GroupingStrategy::default() - ) - ); - assert_eq!( - input, - &SummaryUpdate::column(ColumnRef::Qualified { - table: "metrics".into(), - name: "service".into(), - }) - ); - assert_eq!(reduction, &Reduction::by(vec![])); -} - -#[tokio::test] -async fn map_projection_export_keeps_the_map_over_an_avg_of_exact_states() { - let pre = lower_sql_dialect( - "SELECT map('job', t.service) AS labels, t.avg_bytes FROM (SELECT service, AVG(bytes) AS avg_bytes FROM metrics GROUP BY service) t WHERE t.avg_bytes > 100", - &catalog(), SqlDialect::ClickhouseSQL, AccuracyTarget::exact(), - ).await.unwrap(); - let root = realize(&pre, AccuracyTarget::exact()); - let dag = physical_asap_dag(&root); - assert!(dag.nodes.iter().any(|node| matches!(&node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Project { cols, .. }) - if cols.iter().any(|item| matches!(&item.expr, ScalarExpr::FunctionCall { name, .. } if name == "map")) - ))); - // The AVG under the map projection is built from exact Sum and Count - // states, divided at read time. - let families: Vec<_> = dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { family, .. }) => Some(family), - _ => None, - }) - .collect(); - assert_eq!( - families, - vec![ - &FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - &FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), - ] - ); -} diff --git a/crates/integration-tests/tests/sql_trailing_window.rs b/crates/integration-tests/tests/sql_trailing_window.rs deleted file mode 100644 index 81a229329..000000000 --- a/crates/integration-tests/tests/sql_trailing_window.rs +++ /dev/null @@ -1,246 +0,0 @@ -//! A SQL trailing window, `WHERE ts >= now() - INTERVAL 'w'` on the table's -//! time index, plans like a PromQL range selector `m[w]`: Pass 2 builds its -//! summaries in tumbling windows, Stage 2 maintains them at ingestion time, -//! a deployment that keeps no raw data still gets a plan, and Stage 3 prices -//! each scan by its window. - -use crate::planner_layering_common; - -use asap_cost::CostInputs; -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{NonASAPOp, Operator, QueryRoot}; -use asap_types::workload::{ - DataArrival, DataWorkload, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, - SourceFacts, SqlDialect, -}; -use planner_layering_common::*; - -/// The 5 and 15 minute distinct counts of W-RollupHeavy's template 9. -const QUERIES: [&str; 2] = [ - "SELECT dst_subnet, COUNT(DISTINCT src_addr) AS value FROM flows \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY dst_subnet", - "SELECT dst_subnet, COUNT(DISTINCT src_addr) AS value FROM flows \ - WHERE ts >= now() - INTERVAL '15 minutes' GROUP BY dst_subnet", -]; -const WINDOWS_MS: [u64; 2] = [300_000, 900_000]; - -/// `flows(ts, dst_subnet, src_addr)`, time index `ts`. -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "flows", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("dst_subnet", DataType::Utf8, false), - Field::plain("src_addr", DataType::Float64, false), - ], - 0, - vec![], - ), - ) -} - -/// Both queries every 60 s, relative ε = 0.05 per value, over 1,000 rows/s. -fn workload() -> PlanningWorkload { - let entry = |query: &str| RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(60_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget { - accuracy_metric: Some(AccuracyMetric::Relative), - accuracy_scope: AccuracyScope::PerValue, - ..AccuracyTarget::max_error(0.05).failure_probability(0.05) - }, - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - }; - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), - query_batch: None, - repeating_queries: Some(QUERIES.iter().map(|q| entry(q)).collect()), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000.0, - }, - sources: [( - "flows".to_string(), - SourceFacts { - max_distinct_per_evaluation: Some(1_000_000), - ..SourceFacts::default() - }, - )] - .into(), - }), - } -} - -async fn roots(workload: &PlanningWorkload) -> Vec { - let mut roots = Vec::new(); - for entry in workload.query_workload.entries() { - let root = lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap_or_else(|e| panic!("{}: {e}", entry.query.0)); - roots.push(QueryRoot::Operator(root)); - } - roots -} - -async fn run(deployment: &DeploymentCapabilities) -> Run { - let workload = workload(); - let roots = roots(&workload).await; - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let compose = |choices: &[usize]| compose_logical_candidate(&local, choices).unwrap(); - let exact = compose(&vec![0; local.targets.len()]); - let choices = local - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|choice| { - matches!( - choice, - HowToComputeAggregateMeasure::Summary( - asap_types::ir::schema::SketchAlgorithm::Kmv - ) - ) - }) - .expect("KMV distinct-count sketch with a failure bound") - }) - .collect::>(); - let summary = compose(&choices); - let grids = workload - .query_workload - .entries() - .map(|entry| entry.evaluation_grid_ms()) - .collect::>(); - let tumbling_windows = tumbling_window_candidates(&summary, &grids); - assert_eq!(tumbling_windows.len(), WINDOWS_MS.len()); - let mut candidates = vec![exact, summary.clone()]; - candidates.extend( - window_composition_pass2_candidates(&summary, &tumbling_windows) - .unwrap() - .into_iter() - .filter(|candidate| { - candidate.tumbling_window_candidates.len() == tumbling_windows.len() - }) - .map(|candidate| candidate.roots.unwrap()), - ); - let settings = CostInputs::default(); - run_explicit_candidates( - &workload, - roots, - candidates, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(deployment), - ) -} - -/// Pass 2 offers each query's distinct-count sketch in tumbling windows, and -/// Stage 2 maintains a tumbling-window chain at ingestion time, as for `m[5m]`. -#[tokio::test] -async fn sql_windows_get_tumbling_windows_and_ingestion_time_builds() { - let run = run(&DeploymentCapabilities::default()).await; - for (query, window_ms) in WINDOWS_MS.iter().enumerate() { - let tumbling = run.logical.iter().any(|c| { - sketch_builds(&c.dag).iter().any(|(build, _, _)| { - readers(&c.dag, &c.query_roots, *build).contains(&query) - && matches!(window_form(&c.dag, *build), WindowForm::Tumbling { .. }) - }) - }); - assert!( - tumbling, - "query {query} ({window_ms} ms) has no tumbling-window candidate" - ); - } - let at_ingestion = run.physical.iter().any(|p| { - p.dag().nodes.iter().any(|n| { - matches!( - n.payload, - Operator::ASAP(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) && runs_at_ingestion(p, n.id) - }) - }); - assert!( - at_ingestion, - "no tumbling-window chain maintained at ingestion time" - ); -} - -/// Without raw data at query time, the ingestion-time builds still answer -/// both queries, so the workload has a plan (`run_stages_with` panics when -/// none is selected) and it reads no raw scan at query time. -#[tokio::test] -async fn sql_windows_have_a_plan_without_raw_data() { - let deployment = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let run = run(&deployment).await; - let selected = run.physical(&run.selection.selected); - for n in &selected.dag().nodes { - if matches!(n.payload, Operator::NonASAP(NonASAPOp::Scan { .. })) { - assert!(runs_at_ingestion(selected, n.id), "{}", selected.id); - } - } -} - -/// Each query-time scan is priced by the window above it: the 15 minute -/// query's scan reads 3x the rows of the 5 minute one. -#[tokio::test] -async fn sql_scans_are_priced_by_their_window() { - let run = run(&DeploymentCapabilities::default()).await; - let mut scanned_rows = std::collections::BTreeSet::new(); - for (id, cost) in &run.selection.costs { - let p = run.physical(id); - for (node, node_cost) in &cost.per_node { - let is_query_time_scan = p.dag().nodes.iter().any(|n| { - n.id == *node - && matches!(n.payload, Operator::NonASAP(NonASAPOp::Scan { .. })) - && !runs_at_ingestion(p, n.id) - }); - if !is_query_time_scan { - continue; - } - let rows: u64 = node_cost - .detail - .strip_prefix("scan ") - .and_then(|d| d.split_whitespace().next()) - .and_then(|n| n.parse().ok()) - .unwrap_or_else(|| panic!("scan detail: {}", node_cost.detail)); - scanned_rows.insert(rows); - } - } - // 1 row per ms over each window. - for window_ms in WINDOWS_MS { - assert!( - scanned_rows.contains(&window_ms), - "no scan of {window_ms} rows: {scanned_rows:?}" - ); - } -} diff --git a/crates/integration-tests/tests/sql_two_percentiles.rs b/crates/integration-tests/tests/sql_two_percentiles.rs deleted file mode 100644 index 6ae9ff645..000000000 --- a/crates/integration-tests/tests/sql_two_percentiles.rs +++ /dev/null @@ -1,555 +0,0 @@ -//! Repeating SQL percentiles over one table, every 10 s at ε = 0.01. Q1, -//! the overall p50 and p90 over the last 5 minutes: every candidate that -//! computes one of its percentiles exactly and the other from a sketch -//! compiles, is priced, and answers the exact one as `percentile_cont` does -//! and the sketched one within the sketch's bound. With Q2, the p90 per -//! city over the last 20 minutes, Q1 and Q2 share the table scan, and Q2's -//! per-city tumbling-window chain serves Q1 through its newest tumbling windows rolled up across -//! cities. - -use crate::physical_common; -use crate::planner_layering_common; - -use asap_cost::CostInputs; -use asap_executor::values::Value; -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot}; -use asap_types::workload::{PlanningWorkload, SqlDialect}; -use planner_layering_common::*; -use std::rc::Rc; - -const Q1: &str = "SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY value), \ - percentile_cont(0.9) WITHIN GROUP (ORDER BY value) FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE"; -const Q2: &str = "SELECT city, percentile_cont(0.9) WITHIN GROUP (ORDER BY value) \ - FROM metrics WHERE ts > now() - INTERVAL '20' MINUTE GROUP BY city"; -/// Every query's accuracy target. -const EPSILON: f64 = 0.01; -const MINUTE_MS: i64 = 60_000; -/// The evaluation time of every run. -const EVALUATION_TIME_MS: i64 = 1_800_000_000_000; - -/// `metrics(ts, city, value)`, time index `ts`. -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "metrics", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - ) -} - -/// `queries`, each every 10 s at ε = [`EPSILON`], over `metrics` ingesting -/// 666.67 rows per second. -fn workload(queries: &[&str]) -> PlanningWorkload { - let entry = |query: &&str| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {{"max_error": {EPSILON}, "failure_probability": 0.001}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }; - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": {{"sql": "datafusion_sql"}}, "query_batch": null, - "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 666.67}}}}, - "sources": {{"metrics": {{"series_count": 100}}}}}}}}"#, - queries.iter().map(entry).collect::>().join(", ") - )) - .unwrap() -} - -/// Every stage over `queries`. -async fn run(queries: &[&str]) -> Run { - let workload = workload(queries); - let mut roots = Vec::new(); - for entry in workload.query_workload.entries() { - let root = lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap_or_else(|e| panic!("{}: {e}", entry.query.0)); - roots.push(QueryRoot::Operator(root)); - } - // Every output accepted whatever its guarantee: this test checks the - // answers against the bound. - let settings = CostInputs { - ..CostInputs::default() - }; - let candidates = stage1(queries) - .await - .into_iter() - .map(|(_, roots)| roots) - .collect(); - run_explicit_candidates( - &workload, - roots, - candidates, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(true), - ) -} - -/// Two rows a second over the 20 minutes before the evaluation: city -/// `c{i % 3}`, or none for every seventh row, and a heavy-tailed value from -/// a fixed linear congruential generator. -fn rows() -> Vec<(i64, Option, f64)> { - let mut state: u64 = 0x2545_f491_4f6c_dd1d; - let mut next = move || { - state = state - .wrapping_mul(6_364_136_223_846_793_005) - .wrapping_add(1_442_695_040_888_963_407); - (state >> 33) as f64 / (1u64 << 31) as f64 - }; - (0..20 * 60 * 2) - .map(|i: i64| { - let ts = EVALUATION_TIME_MS - 20 * MINUTE_MS + (i + 1) * 500; - let city = (i % 7 != 0).then(|| format!("c{}", i % 3)); - (ts, city, 10.0 / (1.0 - next()).max(1e-6).sqrt()) - }) - .collect() -} - -/// The values of [`rows`] in `(T - window, T]` of the city `city` (every -/// city for `None`), sorted. -fn values_in(window_ms: i64, city: Option<&str>) -> Vec { - let mut values: Vec = rows() - .into_iter() - .filter(|(ts, row_city, _)| { - *ts > EVALUATION_TIME_MS - window_ms - && city.is_none_or(|c| row_city.as_deref() == Some(c)) - }) - .map(|(_, _, value)| value) - .collect(); - values.sort_by(f64::total_cmp); - values -} - -/// SQL `percentile_cont` of sorted `values`. -fn percentile_cont(q: f64, values: &[f64]) -> f64 { - let rank = q * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - values[low] + (values[high] - values[low]) * (rank - low as f64) -} - -/// Whether `answer` is the `q` percentile of sorted `values` exactly, or -/// within `algorithm`'s bound: a KLL's rank error, a DDSketch's relative -/// value error. -fn within_bound(algorithm: Option<&SketchAlgorithm>, q: f64, answer: f64, values: &[f64]) -> bool { - let exact = percentile_cont(q, values); - if (answer - exact).abs() <= 1e-9 * exact.abs() { - return true; - } - match algorithm { - Some(SketchAlgorithm::Kll) => { - let n = values.len() as f64; - let below = values.partition_point(|&v| v < answer) as f64 / n; - let at_most = values.partition_point(|&v| v <= answer) as f64 / n; - below - EPSILON - 1.0 / n <= q && q <= at_most + EPSILON + 1.0 / n - } - Some(SketchAlgorithm::DDSketch) => (answer - exact).abs() <= EPSILON * exact.abs(), - _ => false, - } -} - -/// `root`'s answer rows at [`EVALUATION_TIME_MS`] over [`rows`], every node at -/// query time, each tumbling-window chain as its tumbling windows. -fn answers(root: &Rc) -> Vec> { - answers_over(root, rows()) -} - -/// [`answers`] over the table rows `table` (time, city, value). -fn answers_over( - root: &Rc, - table: Vec<(i64, Option, f64)>, -) -> Vec> { - let table = table - .into_iter() - .map(|(ts, city, value)| { - vec![ - Value::Timestamp(ts), - city.map_or(Value::Null, |city| Value::Utf8(city.into())), - Value::Float64(value), - ] - }) - .collect(); - // The executor runs a tumbling-window chain as its tumbling windows. - let root = asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains(root) - .unwrap(); - physical_common::execute_raw_rows_at(&root, table, EVALUATION_TIME_MS) -} - -fn as_f64(value: &Value) -> f64 { - match value { - Value::Float64(value) => *value, - other => panic!("not a Float64: {other:?}"), - } -} - -/// The sketch algorithm `root` reads, if any. -fn sketch_of(root: &Rc) -> Option { - OperatorNode::reachable(root) - .into_iter() - .find_map(|node| match node.asap() { - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => Some(*kind.algorithm()), - _ => None, - }) -} - -fn operator(root: &QueryRoot) -> &Rc { - match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator root expected"), - } -} - -/// Q1 with one percentile exact and the other from a sketch: every such -/// plan compiles and is priced (an exact aggregate split from a sketched -/// measure names its one output), and answers each percentile exactly or -/// within the sketch's bound. -#[tokio::test] -async fn q1_with_one_exact_and_one_sketched_percentile_compiles_and_matches() { - let run = run(&[Q1]).await; - let mixed: Vec<&Logical> = run - .logical - .iter() - .filter(|c| { - let reached = OperatorNode::reachable(operator(&c.roots[0])); - reached - .iter() - .any(|n| matches!(n.non_asap(), Some(NonASAPOp::Aggregate { .. }))) - && reached - .iter() - .any(|n| matches!(n.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - }) - .collect(); - assert!(!mixed.is_empty(), "no mixed exact and sketch plan for Q1"); - for candidate in &mixed { - assert!( - run.physical_of(candidate) - .any(|p| run.cost(&p.id).is_some()), - "{}: no physical candidate priced: {:#?}", - candidate.id, - run.selection - .rejected - .iter() - .filter(|r| run.physical_of(candidate).any(|p| p.id == r.id)) - .map(|r| format!("{}: {:?} {}", r.id, r.cause, r.reason)) - .collect::>() - ); - } - let values = values_in(5 * MINUTE_MS, None); - for algorithm in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let candidate = mixed - .iter() - .find(|c| sketch_of(operator(&c.roots[0])) == Some(algorithm)) - .unwrap_or_else(|| panic!("no mixed {algorithm:?} plan")); - let root = operator(&candidate.roots[0]); - // The percentile the exact aggregate computes. - let exact_q = OperatorNode::reachable(root) - .iter() - .find_map(|n| match n.non_asap() { - Some(NonASAPOp::Aggregate { measures, .. }) => match measures.as_slice() { - [AggIntent::Quantile { q, .. }] => Some(*q), - _ => None, - }, - _ => None, - }) - .expect("one exact percentile"); - let rows = answers(root); - assert_eq!(rows.len(), 1, "{}", candidate.id); - for (column, q) in [0.5, 0.9].into_iter().enumerate() { - let answer = as_f64(&rows[0][column]); - let bound = (q != exact_q).then_some(&algorithm); - assert!( - within_bound(bound, q, answer, &values), - "{}: p{} = {answer}, exact {}", - candidate.id, - q * 100.0, - percentile_cont(q, &values) - ); - } - } -} - -/// The id and query roots of every Stage 1 candidate of `queries`. -async fn stage1(queries: &[&str]) -> Vec<(String, Vec)> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = workload(queries); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let mut roots = Vec::new(); - for entry in &entries { - roots.push(QueryRoot::Operator( - lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap(), - )); - } - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - // Explicit fixture configurations: exact; one local substitution at a - // time (mixed percentiles); and all quantiles using each supported family. - // Do not enumerate a Cartesian product of complete workload choices. - let mut choices = std::collections::BTreeSet::from([vec![0; local.targets.len()]]); - for (target, options) in local.targets.iter().enumerate() { - for option in 1..options.choice_count() { - let mut choice = vec![0; local.targets.len()]; - choice[target] = option; - choices.insert(choice); - } - } - for algorithm in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let together: Option> = local.targets.iter().map(|target| target.ways_to_compute.iter() - .position(|candidate| matches!(candidate, HowToComputeAggregateMeasure::Summary(family) if *family == algorithm))).collect(); - if let Some(choice) = together { - choices.insert(choice); - } - } - let share = |roots: Vec| { - asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .into_iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, node), - _ => unreachable!(), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect::>() - }; - let grids = entries - .iter() - .map(|entry| entry.evaluation_grid_ms()) - .collect::>(); - let mut candidates = Vec::new(); - for choice in choices { - let roots = share(compose_logical_candidate(&local, &choice).unwrap()); - candidates.push(roots.clone()); - let tumbling_windows = tumbling_window_candidates(&roots, &grids); - if !tumbling_windows.is_empty() { - candidates.extend( - window_composition_pass2_candidates(&roots, &tumbling_windows) - .unwrap() - .into_iter() - .filter(|candidate| { - candidate.tumbling_window_candidates.len() == tumbling_windows.len() - }) - .filter_map(|candidate| candidate.roots.ok()) - .map(share), - ); - } - } - candidates - .into_iter() - .enumerate() - .map(|(index, roots)| (format!("explicit-{}", index + 1), roots)) - .collect() -} - -/// Q1 and Q2 read the same rows of `metrics`: a candidate reads them with -/// one scan. A table scan has no unique key, and is shared all the same: -/// each reader reads every row it outputs. -#[tokio::test] -async fn q1_and_q2_share_the_table_scan() { - let scans = |root: &QueryRoot| -> Vec<*const OperatorNode> { - OperatorNode::reachable(operator(root)) - .iter() - .filter(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .map(Rc::as_ptr) - .collect() - }; - let candidates = stage1(&[Q1, Q2]).await; - assert!( - candidates.iter().any(|(_, roots)| { - let q2_scans = scans(&roots[1]); - scans(&roots[0]).iter().any(|scan| q2_scans.contains(scan)) - }), - "no candidate shares the scan" - ); -} - -/// Whether Q2's per-city tumbling-window chain (120 tumbling windows of 10 s) serves both of -/// Q1's percentiles in `roots`: Q1 merges each city's newest 30 tumbling windows (its -/// 5 minutes), then rolls the cities up into one state, which both of its -/// estimates read. -fn newest_tumbling_windows_rolled_up(roots: &[QueryRoot]) -> bool { - let (q1, q2) = (operator(&roots[0]), operator(&roots[1])); - let q2_chains: Vec<*const OperatorNode> = OperatorNode::reachable(q2) - .iter() - .filter(|n| { - matches!( - n.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count: 120, - .. - }) - ) - }) - .map(Rc::as_ptr) - .collect(); - let rollups: Vec> = OperatorNode::reachable(q1) - .into_iter() - .filter(|n| match n.asap() { - Some(ASAPOp::SummaryMerge { - children, - group_by: Some(keys), - newest_tumbling_windows: None, - }) => { - keys.is_empty() - && matches!( - children[0].asap(), - Some(ASAPOp::SummaryMerge { - children: chain, - newest_tumbling_windows: Some(30), - group_by: None, - }) if q2_chains.contains(&Rc::as_ptr(&chain[0])) - ) - } - _ => false, - }) - .collect(); - OperatorNode::reachable(q1) - .iter() - .filter(|n| match n.asap() { - Some(ASAPOp::SummaryEstimate { summary_input, .. }) => { - rollups.iter().any(|r| Rc::ptr_eq(r, summary_input)) - } - _ => false, - }) - .count() - == 2 -} - -/// The values of [`rows`] of city `city` (no city for `None`) in the last -/// 20 minutes, sorted. -fn city_values(city: Option<&str>) -> Vec { - let mut values: Vec = rows() - .into_iter() - .filter(|(ts, row_city, _)| { - *ts > EVALUATION_TIME_MS - 20 * MINUTE_MS && row_city.as_deref() == city - }) - .map(|(_, _, value)| value) - .collect(); - values.sort_by(f64::total_cmp); - values -} - -/// Q2's per-city chain serves both of Q1's percentiles through its newest -/// 30 tumbling windows rolled up across cities. Each query's answers are within the -/// sketch's bound of the exact `percentile_cont`, and over a window with -/// no row Q1 still answers one NULL row, as the global SQL aggregate the -/// rollup stands for. -#[tokio::test] -async fn q2_tumbling_window_chain_serves_q1_through_newest_tumbling_windows_and_a_rollup() { - let candidates = stage1(&[Q1, Q2]).await; - for algorithm in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let (id, roots) = candidates - .iter() - .find(|(_, roots)| { - newest_tumbling_windows_rolled_up(roots) - && sketch_of(operator(&roots[0])) == Some(algorithm) - }) - .unwrap_or_else(|| panic!("no {algorithm:?} newest-tumbling-windows rollup candidate")); - let q1 = answers(operator(&roots[0])); - let values = values_in(5 * MINUTE_MS, None); - assert_eq!(q1.len(), 1, "{id}"); - for (column, q) in [0.5, 0.9].into_iter().enumerate() { - let answer = as_f64(&q1[0][column]); - assert!( - within_bound(Some(&algorithm), q, answer, &values), - "{id}: Q1 p{} = {answer}, exact {}", - q * 100.0, - percentile_cont(q, &values) - ); - } - let older = rows() - .into_iter() - .filter(|(ts, _, _)| *ts <= EVALUATION_TIME_MS - 5 * MINUTE_MS) - .collect(); - let empty = answers_over(operator(&roots[0]), older); - assert!( - matches!(empty.as_slice(), [row] if matches!(row.as_slice(), [Value::Null, Value::Null])), - "{id}: {empty:?}" - ); - let q2 = answers(operator(&roots[1])); - assert_eq!(q2.len(), 4, "{id}: three cities and none"); - for row in q2 { - let city = match &row[0] { - Value::Utf8(city) => Some(city.to_string()), - Value::Null => None, - other => panic!("city: {other:?}"), - }; - let values = city_values(city.as_deref()); - let answer = as_f64(&row[1]); - assert!( - within_bound(Some(&algorithm), 0.9, answer, &values), - "{id}: Q2 p90 of {city:?} = {answer}, exact {}", - percentile_cont(0.9, &values) - ); - } - } -} - -/// A global SQL `COUNT(*)` is 0 over no rows, but a rollup of the per-city -/// counts would sum no count into NULL. A rollup to no keys over a table is -/// offered, yet no candidate answers the global count from the per-city -/// counts: an exact rollup never sums Int64 counts again. -#[tokio::test] -async fn a_global_count_is_not_rolled_up_from_grouped_counts() { - let global = "SELECT COUNT(*) FROM metrics WHERE ts > now() - INTERVAL '5' MINUTE"; - let by_city = "SELECT city, COUNT(*) FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE GROUP BY city"; - let finalized = |root: &QueryRoot| -> Vec<*const OperatorNode> { - OperatorNode::reachable(operator(root)) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. }))) - .map(Rc::as_ptr) - .collect() - }; - for (id, roots) in stage1(&[global, by_city]).await { - let by_city_values = finalized(&roots[1]); - assert!( - !finalized(&roots[0]) - .iter() - .any(|value| by_city_values.contains(value)), - "{id}: the global count reads the per-city counts" - ); - } -} diff --git a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs deleted file mode 100644 index 25117bb35..000000000 --- a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs +++ /dev/null @@ -1,773 +0,0 @@ -//! Frontends → Pass 1 → composed candidate logical ASAP DAGs: every candidate of -//! a query is a valid DAG with the query's output columns (names, types and -//! nullability). - -use std::rc::Rc; -use std::sync::Arc; - -use asap_executor::{ - physical_planner::compile, - runtime::{Limits, RunContext, Scope}, - sources::InMemoryDataStore, - values::{Batch, Value}, -}; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, -}; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{ - ExactKind, FieldDataType, SketchAlgorithm, SummaryInputExpr, SummaryUpdate, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::stages::HowToComputeAggregateMeasure; -use asap_types::workload::{ - BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Query, QueryLanguage, - QueryRequirements, QueryWorkload, -}; -use futures::StreamExt; - -use crate::physical_common; - -fn accuracy() -> AccuracyTarget { - AccuracyTarget::max_error(0.01).failure_probability(0.01) -} - -async fn sql(query: &str) -> QueryRoot { - let catalog = SqlCatalog::new().with_table( - "flows", - Schema::new(vec![ - Field::plain("src", DataType::Utf8, false), - Field::plain("dst", DataType::Utf8, false), - Field::plain("bytes", DataType::Float64, false), - ]), - ); - let root = lower_sql(query, &catalog, accuracy()) - .await - .unwrap_or_else(|e| panic!("lower failed for {query:?}: {e}")); - QueryRoot::Operator(root) -} - -fn promql(query: &str) -> QueryRoot { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy: accuracy(), - ..Default::default() - }, - predictability: Default::default(), - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1000), - )), - }; - asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap_or_else(|e| panic!("lower failed for {query:?}: {e}")) - .remove(0) -} - -fn output_columns(root: &QueryRoot) -> Vec<(String, String, bool)> { - match root { - QueryRoot::Operator(node) => node - .schema - .fields - .iter() - .map(|field| { - ( - field.name.clone(), - format!("{:?}", field.dtype), - field.nullable, - ) - }) - .collect(), - QueryRoot::Scalar(_) => vec![], - } -} - -/// Compose every candidate of `root`; returns how many there were. -fn compose_all(label: &str, root: QueryRoot) -> usize { - let expected = output_columns(&root); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[root]), - ) - .unwrap(); - let radix: Vec<_> = inventory - .targets - .iter() - .map(|target| target.choice_count()) - .collect(); - let total: usize = radix.iter().product(); - let mut choice = vec![0; radix.len()]; - for _ in 0..total { - let composed = compose_logical_candidate(&inventory, &choice) - .unwrap_or_else(|e| panic!("{label} {choice:?}: {e}")); - let root = &composed[0]; - root.validate_structure() - .unwrap_or_else(|e| panic!("{label} {choice:?}: {e}")); - let columns = output_columns(root); - // A heap top-k answers the declared `[keys.., topk_k]` with one row - // per ranked item: `topk_k` its Float64 estimate (the value the - // exact top-k's physical plan answers too), then the item labels. - let heap_topk = expected - .last() - .is_some_and(|(name, _, _)| name.starts_with("topk_")) - && columns.get(expected.len() - 1).map(|c| c.1.as_str()) == Some("Plain(Float64)"); - if heap_topk { - let (ranked, keys) = expected.split_last().unwrap(); - assert_eq!(&columns[..keys.len()], keys, "{label} {choice:?}"); - assert_eq!(columns[keys.len()].0, ranked.0, "{label} {choice:?}"); - } else { - assert_eq!(columns, expected, "{label} {choice:?}"); - } - for (digit, radix) in choice.iter_mut().zip(&radix).rev() { - *digit += 1; - if *digit < *radix { - break; - } - *digit = 0; - } - } - total -} - -#[tokio::test] -async fn sql_candidates_compose() { - for query in [ - "SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a'", - "SELECT src, count(*) FROM flows GROUP BY src HAVING count(*) > 10", - "SELECT src, avg(bytes), count(*), approx_distinct(dst) FROM flows GROUP BY src", - "SELECT count(*) FILTER (WHERE bytes > 10), approx_percentile_cont(bytes, 0.99) FROM flows", - "SELECT avg(bytes) FROM flows", - "SELECT sum(flows.bytes) FROM flows", - "SELECT src, approx_percentile_cont(bytes, 0.99) FILTER (WHERE bytes > 10) FROM flows GROUP BY src", - "SELECT approx_percentile_cont(bytes, 0.99) FILTER (WHERE bytes > 10) FROM flows", - ] { - assert!(compose_all(query, sql(query).await) >= 1, "{query}"); - } -} - -#[test] -fn promql_candidates_compose() { - for query in [ - "avg by (job) (m)", - "avg_over_time(m[5m])", - "quantile(0.99, m) - quantile(0.5, m)", - "scalar(quantile(0.99, m)) - scalar(quantile(0.5, m))", - "topk by (job) (10, sum_over_time(m[1m]))", - "topk(3, sum by (job) (rate(m[5m])))", - "count by (job) (m)", - "count_over_time(m[5m])", - "quantile_over_time(0.99, m[5m])", - ] { - assert!(compose_all(query, promql(query)) >= 1, "{query}"); - } -} - -/// The summary estimates of `root`'s candidates that vary one target's -/// choice, every other target taking its first. -fn composed_estimates(root: QueryRoot) -> Vec> { - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[root]), - ) - .unwrap(); - let targets = inventory.targets.len(); - let choices: Vec<_> = inventory - .targets - .iter() - .enumerate() - .flat_map(|(target, computation)| { - (1..computation.choice_count()).map(move |index| { - let mut choice = vec![0; targets]; - choice[target] = index; - choice - }) - }) - .collect(); - choices - .into_iter() - .flat_map(|choice| { - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(node) = &composed[0] else { - panic!("operator root expected") - }; - OperatorNode::reachable(node) - }) - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - .collect() -} - -fn update_of(estimate: &OperatorNode) -> (SketchAlgorithm, SummaryUpdate) { - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = estimate.asap() else { - unreachable!() - }; - let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - input, - .. - }) = summary_input.asap() - else { - panic!("sketch state expected") - }; - (*kind.algorithm(), input.clone()) -} - -/// A heap sketch's bound is per item frequency, not Top-K membership, and -/// its weights are not proven non-negative: a Top-K estimate gets no -/// guarantee. A unit-count CMS estimate keeps its bound. -#[test] -fn topk_estimates_carry_no_frequency_bound() { - let topk = composed_estimates(promql("topk by (job) (10, sum_over_time(m[1m]))")); - let algorithms: Vec<_> = topk.iter().map(|node| update_of(node).0).collect(); - assert_eq!( - algorithms, - [ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap - ] - ); - assert!(topk.iter().all(|node| node.guarantee.is_none())); - - let count = composed_estimates(promql("count by (job) (m)")); - let cms = count - .iter() - .find(|node| update_of(node).0 == SketchAlgorithm::Cms) - .expect("CMS candidate"); - assert!(cms.guarantee.as_ref().is_some_and(|g| !g.is_exact())); -} - -/// Top-K over a grouped sum is a whole-expression heap: it absorbs the -/// sum and ranks its group keys, read from the rows the sum reads. -#[test] -fn topk_over_an_aggregate_ranks_its_labels() { - let estimates = composed_estimates(promql("topk(3, sum by (job) (rate(m[5m])))")); - assert_eq!(estimates.len(), 2); - for estimate in &estimates { - assert_eq!( - update_of(estimate).1.item, - Some(SummaryInputExpr::Column(ColumnRef::Named("job".into()))) - ); - } -} - -/// A subquery inside a measure filter is substituted too: each choice for -/// the inner quantile gives its own DAG, and an approximate inner result clears -/// the outer measure's guarantee. -#[tokio::test] -async fn subquery_in_a_measure_filter_is_substituted() { - let query = "SELECT src, count(*) FILTER (WHERE bytes > (SELECT approx_percentile_cont(bytes, 0.5) FROM flows)) \ - FROM flows GROUP BY src"; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[sql(query).await]), - ) - .unwrap(); - assert_eq!(inventory.targets.len(), 2); - let inner = inventory - .targets - .iter() - .position(|target| { - inventory - .aggregate(target) - .non_asap() - .and_then(|op| match op { - NonASAPOp::Aggregate { reduction, .. } => reduction.group_keys(), - _ => None, - }) - .is_some_and(|by| by.keys().is_empty()) - }) - .expect("inner quantile"); - let candidates = &inventory.targets[inner].ways_to_compute; - assert!(candidates.len() > 1); - let (mut seen, mut approximate_inner) = (Vec::new(), false); - for (index, candidate) in candidates.iter().enumerate() { - let mut choice = vec![0; 2]; - choice[inner] = index; - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root expected") - }; - let shown = format!("{root:?}"); - assert!(!seen.contains(&shown), "{candidate:?} composes as another"); - seen.push(shown); - let finalize = OperatorNode::reachable(root) - .into_iter() - .find(|node| { - matches!( - node.asap(), - Some(ASAPOp::FinalizeExactAccumulator { child }) - if matches!(child.asap(), Some(ASAPOp::SummaryAgg { filter: Some(_), .. })) - ) - }) - .expect("filtered outer count"); - let Some(ASAPOp::FinalizeExactAccumulator { child }) = finalize.asap() else { - unreachable!() - }; - let exact = |node: &OperatorNode| node.guarantee.as_ref().is_some_and(|g| g.is_exact()); - // The state's children: its input rows, then the subquery. - let inputs_exact = child.children().into_iter().all(|node| exact(node)); - assert_eq!(exact(&finalize), inputs_exact, "{candidate:?}"); - approximate_inner |= !inputs_exact; - } - assert!(approximate_inner); -} - -/// Run the logical DAG `root` over `rows` of its one scanned table, each row -/// given by column name. Rows come back shown and sorted. -fn execute(root: &Rc, rows: &[Vec<(&str, Value)>]) -> Vec> { - let dag = physical_common::compile_physical_asap_dag(root).unwrap(); - let scan = dag - .nodes - .iter() - .find(|node| { - matches!( - &node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { .. }) - ) - }) - .expect("one scan"); - let PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { source, .. }) = &scan.payload else { - unreachable!() - }; - let schema = Arc::new(scan.output_schema.clone()); - let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - let rows = rows - .iter() - .map(|row| { - schema - .fields - .iter() - .map(|field| { - row.iter() - .find(|(name, _)| *name == field.name) - .map(|(_, value)| value.clone()) - .expect("value for every column") - }) - .collect() - }) - .collect(); - let store = InMemoryDataStore::default() - .with_raw_data( - source.clone(), - schema.clone(), - vec![Batch::try_new(schema.clone(), rows).unwrap()], - ) - .unwrap(); - let bound = plan.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: 0, - }, - Limits::default(), - ) - .unwrap(); - let show = |value: &Value| match value { - Value::Null => "NULL".to_string(), - Value::Bool(value) => value.to_string(), - Value::Float64(value) => value.to_string(), - Value::Utf8(value) => format!("{value:?}"), - other => format!("{other:?}"), - }; - let mut out: Vec> = futures::executor::block_on(async { - let mut stream = bound.execute(plan.roots(), context).unwrap().remove(0); - let mut rows = Vec::new(); - while let Some(batch) = stream.next().await { - rows.extend( - batch - .unwrap() - .to_rows() - .iter() - .map(|row| row.iter().map(show).collect()), - ); - } - rows - }); - out.sort(); - out -} - -/// SQL `avg` from its `sum` and `count` states, composed for the query. -async fn avg_as_sum_and_count( - catalog: &SqlCatalog, - query: &str, -) -> (Rc, Rc) { - let exact = lower_sql(query, catalog, accuracy()).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(exact.clone()), - ]), - ) - .unwrap(); - let choice: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - *c == HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum, - ExactKind::Count, - ]) - }) - .expect("avg as sum and count") - }) - .collect(); - let composed = - compose_logical_candidate(&inventory, &choice).unwrap_or_else(|e| panic!("{query}: {e}")); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root expected") - }; - assert!( - OperatorNode::reachable(root) - .iter() - .any(|node| matches!(node.non_asap(), Some(NonASAPOp::Join { .. }))), - "{query}: sum joined with count expected" - ); - assert_eq!( - output_columns(&QueryRoot::Operator(root.clone())), - output_columns(&QueryRoot::Operator(exact.clone())), - "{query}" - ); - (exact, root.clone()) -} - -fn avg_table() -> SqlCatalog { - SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("k", DataType::Utf8, true), - Field::plain("ok", DataType::Bool, false), - Field::plain("value", DataType::Float64, false), - ]), - ) -} - -/// SQL `avg` as `sum` and `count` states executes as the exact aggregate: -/// NULL and '' groups stay apart, a non-Utf8 key works, and an empty global -/// input averages to NULL. -#[tokio::test] -async fn sql_avg_from_sum_and_count_executes_as_avg() { - let catalog = avg_table(); - let row = |k: Option<&str>, ok: bool, value: f64| { - vec![ - ("k", k.map_or(Value::Null, |k| Value::Utf8(k.into()))), - ("ok", Value::Bool(ok)), - ("value", Value::Float64(value)), - ] - }; - let rows = vec![ - row(None, true, 2.), - row(None, true, 4.), - row(Some(""), false, 10.), - row(Some("a"), false, 200.), - row(Some("a"), true, 1.), - ]; - let rows_of = |rows: &[&[&str]]| -> Vec> { - rows.iter() - .map(|row| row.iter().map(|cell| cell.to_string()).collect()) - .collect() - }; - for (query, input, want) in [ - ( - "SELECT k, avg(value) FROM t GROUP BY k", - rows.clone(), - rows_of(&[&["\"\"", "10"], &["\"a\"", "100.5"], &["NULL", "3"]]), - ), - ( - "SELECT ok, avg(value) FROM t GROUP BY ok", - rows.clone(), - rows_of(&[&["false", "105"], &["true", "2.3333333333333335"]]), - ), - ( - "SELECT avg(value) FROM t", - rows.clone(), - rows_of(&[&["43.4"]]), - ), - ("SELECT avg(value) FROM t", vec![], rows_of(&[&["NULL"]])), - ] { - let (exact, root) = avg_as_sum_and_count(&catalog, query).await; - assert_eq!(execute(&exact, &input), want, "{query}"); - assert_eq!(execute(&root, &input), want, "{query}"); - } -} - -/// A filtered SQL `avg` is NULL where no row passes the filter: its quotient -/// is NULL when the count is 0. The runtime executes no filtered state yet, -/// so the shape is checked. -#[tokio::test] -async fn filtered_sql_avg_is_null_without_matching_rows() { - let catalog = avg_table(); - for query in [ - "SELECT k, avg(value) FILTER (WHERE value > 100) FROM t GROUP BY k", - "SELECT avg(value) FILTER (WHERE value > 100) FROM t", - ] { - let (_, root) = avg_as_sum_and_count(&catalog, query).await; - assert!(root.schema.fields.last().unwrap().nullable, "{query}"); - let null_when_no_count = OperatorNode::reachable(&root).iter().any(|node| { - let Some(NonASAPOp::Project { cols, child, .. }) = node.non_asap() else { - return false; - }; - let count = child.schema.fields.len() - 1; - matches!( - &cols.last().unwrap().expr, - ScalarExpr::Case { branches, .. } if matches!( - &branches[..], - [(ScalarExpr::Compare { left, op: CompareOpKind::Eq, right, .. }, - ScalarExpr::Literal(ScalarValue::Null))] - if **left == ScalarExpr::Column(count) - && **right == ScalarExpr::Literal(ScalarValue::Int64(0)) - ) - ) - }); - assert!(null_when_no_count, "{query}: {root:#?}"); - } -} - -/// PromQL `avg` divides its `sum` and `count` vectors with a `BinaryOp`. -#[test] -fn promql_avg_divides_sum_and_count_vectors() { - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[promql( - "avg by (job) (m)", - )]), - ) - .unwrap(); - assert_eq!( - inventory.targets[0].ways_to_compute[0], - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum, ExactKind::Count]) - ); - let composed = compose_logical_candidate(&inventory, &[0]).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root expected") - }; - let kinds: Vec<_> = OperatorNode::reachable(root) - .iter() - .map(|node| node.operator.kind_name()) - .collect(); - assert!( - kinds.contains(&"BinaryOp") && !kinds.contains(&"Join"), - "{kinds:?}" - ); -} - -/// `t(k Utf8 NULL, )` as a catalog. -fn table(columns: &[(&str, bool)]) -> SqlCatalog { - let mut fields = vec![Field::plain("k", DataType::Utf8, true)]; - fields.extend( - columns - .iter() - .map(|(name, nullable)| Field::plain(*name, DataType::Float64, *nullable)), - ); - SqlCatalog::new().with_table("t", Schema::new(fields)) -} - -/// Every candidate of `query` over `catalog`, composed, by candidate. -async fn every_candidate( - catalog: &SqlCatalog, - query: &str, -) -> Vec<( - HowToComputeAggregateMeasure, - Result, asap_logical_asap_optimization::PerComputationCandidateError>, -)> { - let root = lower_sql(query, catalog, accuracy()).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - assert_eq!(inventory.targets.len(), 1, "{query}"); - inventory.targets[0] - .ways_to_compute - .iter() - .enumerate() - .map(|(index, candidate)| { - ( - candidate.clone(), - compose_logical_candidate(&inventory, &[index]), - ) - }) - .collect() -} - -/// The update of every `count` state (exact or sketch) below `root`. -fn count_updates(root: &QueryRoot) -> Vec { - let QueryRoot::Operator(root) = root else { - panic!("operator root expected") - }; - OperatorNode::reachable(root) - .iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { family, input, .. }) - if !matches!( - family, - FieldDataType::ExactAggregate(asap_types::ir::schema::ExactKind::Sum, _) - ) => - { - Some(input.clone()) - } - _ => None, - }) - .collect() -} - -/// Whether `update` reads only the column `name` of `t`. -fn reads_only(update: &SummaryUpdate, name: &str) -> bool { - let column = SummaryInputExpr::Column(ColumnRef::Qualified { - table: "t".into(), - name: name.into(), - }); - match &update.item { - Some(item) => *item == column && update.weight == SummaryInputExpr::Constant(1.0), - None => update.weight == column, - } -} - -/// `avg(c)`'s count counts `c`, not a nullable `value` column: the result is -/// the exact `avg`. -#[tokio::test] -async fn sql_avg_counts_its_own_input() { - let catalog = table(&[("value", true), ("c", false)]); - let query = "SELECT k, avg(c) FROM t GROUP BY k"; - let (exact, root) = avg_as_sum_and_count(&catalog, query).await; - let row = |k: &str, value: Option, c: f64| { - vec![ - ("k", Value::Utf8(k.into())), - ("value", value.map_or(Value::Null, Value::Float64)), - ("c", Value::Float64(c)), - ] - }; - let rows = vec![ - row("a", None, 1.), - row("a", Some(5.), 3.), - row("b", None, 4.), - ]; - let want = vec![ - vec!["\"a\"".to_string(), "2".to_string()], - vec!["\"b\"".to_string(), "4".to_string()], - ]; - assert_eq!(execute(&exact, &rows), want); - assert_eq!(execute(&root, &rows), want); -} - -/// SQL `avg` as `sum` and `count` keeps an explicit output alias. -#[tokio::test] -async fn sql_avg_keeps_its_explicit_output_name() { - let (_, root) = - avg_as_sum_and_count(&avg_table(), "SELECT avg(value) AS avg_latency FROM t").await; - assert_eq!(root.schema.fields.last().unwrap().name, "avg_latency"); -} - -/// SQL `avg` of a non-null Int64 column as `sum` and `count` keeps the exact plan's Float64 columns. -#[tokio::test] -async fn sql_avg_of_an_int64_column_keeps_the_exact_columns() { - let catalog = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("k", DataType::Utf8, true), - Field::plain("n", DataType::Int64, false), - ]), - ); - let query = "SELECT k, avg(n) FROM t GROUP BY k"; - let (exact, root) = avg_as_sum_and_count(&catalog, query).await; - assert_eq!( - root.schema.fields.last().unwrap().plain_dtype(), - Some(&DataType::Float64) - ); - let row = |k: &str, n: i64| vec![("k", Value::Utf8(k.into())), ("n", Value::Int64(n))]; - let rows = vec![row("a", 1), row("a", 2), row("b", 4)]; - let want = vec![ - vec!["\"a\"".to_string(), "1.5".to_string()], - vec!["\"b\"".to_string(), "4".to_string()], - ]; - assert_eq!(execute(&exact, &rows), want); -} - -/// `count(*)` counts a non-null column, never a nullable one; executed, it -/// counts the rows whose nullable column is NULL too. -#[tokio::test] -async fn sql_count_star_counts_a_non_null_column() { - let catalog = table(&[("v", true), ("n", false)]); - let query = "SELECT k, count(*) FROM t GROUP BY k"; - for (candidate, composed) in every_candidate(&catalog, query).await { - let composed = composed.unwrap_or_else(|e| panic!("{candidate:?}: {e}")); - composed[0].validate_structure().unwrap(); - let updates = count_updates(&composed[0]); - assert_eq!(updates.len(), 1, "{candidate:?}"); - assert!(reads_only(&updates[0], "n"), "{candidate:?}: {updates:?}"); - } - let (exact, root) = { - let exact = lower_sql(query, &catalog, accuracy()).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(exact.clone()), - ]), - ) - .unwrap(); - let QueryRoot::Operator(root) = compose_logical_candidate(&inventory, &[0]) - .unwrap() - .remove(0) - else { - panic!("operator root expected") - }; - (exact, root) - }; - let row = |k: &str, v: Option| { - vec![ - ("k", Value::Utf8(k.into())), - ("v", v.map_or(Value::Null, Value::Float64)), - ("n", Value::Float64(0.)), - ] - }; - let rows = vec![row("a", None), row("a", Some(1.)), row("b", None)]; - assert_eq!(execute(&root, &rows), execute(&exact, &rows)); -} - -/// `count(*)` with no column known to be non-null (one nullable numeric -/// column, or several and none named `value`) is unsupported for every -/// candidate, not a count of a nullable column or an invalid DAG. -#[tokio::test] -async fn sql_count_star_without_a_non_null_column_is_unsupported() { - for columns in [&[("v", true)][..], &[("a", true), ("b", true)][..]] { - let catalog = table(columns); - for (candidate, composed) in - every_candidate(&catalog, "SELECT k, count(*) FROM t GROUP BY k").await - { - match composed { - Err(asap_logical_asap_optimization::PerComputationCandidateError::Unsupported( - _, - )) => {} - other => panic!("{columns:?} {candidate:?}: {other:?}"), - } - } - } -} - -/// `count(v)` of a nullable `v` updates on `v` under its `v IS NOT NULL` -/// filter: it counts the non-null values. The runtime executes no filtered -/// state yet, so the shape is checked. -#[tokio::test] -async fn sql_count_of_a_nullable_column_counts_its_values() { - let catalog = table(&[("v", true), ("n", false)]); - for (candidate, composed) in - every_candidate(&catalog, "SELECT k, count(v) FROM t GROUP BY k").await - { - let composed = composed.unwrap_or_else(|e| panic!("{candidate:?}: {e}")); - composed[0].validate_structure().unwrap(); - let updates = count_updates(&composed[0]); - assert_eq!(updates.len(), 1, "{candidate:?}"); - assert!(reads_only(&updates[0], "v"), "{candidate:?}: {updates:?}"); - } -} diff --git a/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs b/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs deleted file mode 100644 index dbe386cac..000000000 --- a/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! SQL heavy-hitter queries through Stage 1 (every Pass 1 choice composed) -//! and physical ASAP-aware optimization: exact top-k sorts by the ranked -//! measure, descending, whatever the group keys are called or typed, no -//! valid candidate is pruned, and every candidate answers the ranked value -//! then the group keys naming each ranked item. - -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, -}; -use asap_physical_asap_optimization::optimize_physical_asap; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::stages::HowToComputeAggregateMeasure; - -fn approximate() -> AccuracyTarget { - AccuracyTarget::max_error(0.01).failure_probability(0.001) -} - -fn names(node: &OperatorNode) -> Vec { - node.schema.fields.iter().map(|f| f.name.clone()).collect() -} - -/// The `(sort column name, ascending)` of an exact top-k's `Sort`, if `root` -/// is one (a heavy-hitter summary candidate has none). -fn exact_topk_sort(root: &OperatorNode) -> Option<(String, bool)> { - let NonASAPOp::Project { child: limit, .. } = root.non_asap()? else { - return None; - }; - let NonASAPOp::Limit { - n: Some(5), - child: sort, - .. - } = limit.non_asap()? - else { - return None; - }; - let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { - panic!("sort below limit") - }; - let [key] = keys.as_slice() else { - panic!("one sort key") - }; - let ScalarExpr::Column(column) = key.expr else { - panic!("sort by a column") - }; - Some((sort.schema.fields[column].name.clone(), key.ascending)) -} - -/// Every Pass 1 choice of `sql` under `accuracy`, composed and physically -/// optimized: each must succeed, every exact top-k must sort by `ranked` -/// descending, and every root must output `root_names`. Returns how many -/// candidates had an exact top-k. -async fn check_every_choice( - sql: &str, - accuracy: AccuracyTarget, - ranked: &str, - root_names: &[&str], -) -> usize { - let catalog = SqlCatalog::new().with_table( - "metrics", - Schema::new(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("host", DataType::Utf8, false), - Field::plain("port", DataType::Int64, false), - ]), - ); - let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); - let NonASAPOp::Aggregate { measures, .. } = root.expect_non_asap() else { - panic!("{sql}: heavy-hitter aggregate root") - }; - assert!( - matches!(measures.as_slice(), [AggIntent::TopK { k: 5, .. }]), - "{sql}" - ); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - // The cartesian product of every target's candidates. - let mut choices = vec![vec![]]; - for target in &inventory.targets { - choices = choices - .into_iter() - .flat_map(|prefix: Vec| { - (0..target.choice_count()).map(move |i| { - let mut choice = prefix.clone(); - choice.push(i); - choice - }) - }) - .collect(); - } - let mut exact = 0; - for choice in choices { - let composed = match compose_logical_candidate(&inventory, &choice) { - Ok(composed) => composed, - // Stage 1 cannot yet compose a heavy-hitter summary over some - // keys (an integer or several key columns); Stage 2 never sees - // that candidate. Every other choice must compose. - Err(e) => { - let summarized_topk = inventory.targets.iter().zip(&choice).any(|(t, &i)| { - let NonASAPOp::Aggregate { measures, .. } = - inventory.aggregate(t).expect_non_asap() - else { - return false; - }; - matches!(measures[t.measure_index], AggIntent::TopK { .. }) - && t.computation_for(i).map(|(c, _)| c) - != Some(&HowToComputeAggregateMeasure::Exact) - }); - assert!(summarized_topk, "{sql}, choice {choice:?}: {e}"); - continue; - } - }; - let roots: Vec<_> = composed; - let candidate = optimize_physical_asap( - "L1", - &roots, - &[], - &asap_types::workload::DataWorkload::default(), - false, - ) - .unwrap_or_else(|e| panic!("{sql}, choice {choice:?} pruned: {e}")) - .remove(0); - let QueryRoot::Operator(physical) = &candidate.roots[0] else { - panic!("operator root") - }; - assert_eq!(names(physical), root_names, "{sql}, choice {choice:?}"); - if let Some(sort) = exact_topk_sort(physical) { - assert_eq!( - sort, - (ranked.to_string(), false), - "{sql}, choice {choice:?}" - ); - exact += 1; - } - } - exact -} - -/// Each query under an exact and an approximate target; every exact top-k -/// sorts by the ranked measure. -async fn check(sql: &str, ranked: &str, root_names: &[&str]) { - for accuracy in [AccuracyTarget::exact(), approximate()] { - let exact = check_every_choice(sql, accuracy.clone(), ranked, root_names).await; - assert!( - exact > 0, - "{sql} under {accuracy:?}: an exact top-k candidate" - ); - } -} - -/// Grouping by a column named `value` still ranks by the count. -#[tokio::test] -async fn group_key_named_value_ranks_by_count() { - check( - "SELECT value, COUNT(*) AS c FROM metrics GROUP BY value ORDER BY c DESC LIMIT 5", - "count(Int64(1))", - &["topk_5", "value"], - ) - .await; -} - -/// An integer group key leaves no `value` column; the count is still ranked. -#[tokio::test] -async fn integer_group_key_ranks_by_count() { - check( - "SELECT port, COUNT(*) AS c FROM metrics GROUP BY port ORDER BY c DESC LIMIT 5", - "count(Int64(1))", - &["topk_5", "port"], - ) - .await; -} - -/// Two group keys, one of them an integer. -#[tokio::test] -async fn multi_key_ranks_by_count() { - check( - "SELECT host, port, COUNT(*) AS c FROM metrics GROUP BY host, port \ - ORDER BY c DESC LIMIT 5", - "count(Int64(1))", - &["topk_5", "host", "port"], - ) - .await; -} - -/// A SUM ranking sorts by the sum. -#[tokio::test] -async fn sum_ranking_ranks_by_sum() { - check( - "SELECT port, SUM(value) AS s FROM metrics GROUP BY port ORDER BY s DESC LIMIT 5", - "sum(metrics.value)", - &["topk_5", "port"], - ) - .await; -} diff --git a/crates/integration-tests/tests/summary_coverage_e2e.rs b/crates/integration-tests/tests/summary_coverage_e2e.rs deleted file mode 100644 index f2ce4aad3..000000000 --- a/crates/integration-tests/tests/summary_coverage_e2e.rs +++ /dev/null @@ -1,262 +0,0 @@ -//! Query string → planned summary state → `coverage()`, for the examples of -//! the ASAP primitive schema design doc (#573 §4.2.2). Each query is lowered -//! and its logical candidates composed, and the quantile's KLL `SummaryAgg` -//! is checked: what its `definition` keeps and what its `selection` takes, -//! row by row of §4.2.2's "More examples". The tests read the candidates, not -//! plan selection's choice, so they do not depend on which candidate is -//! cheapest. - -use std::ops::Bound; -use std::rc::Rc; - -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::explicit_summary_candidate; -use asap_types::ir::properties::summary_coverage::{ColumnIdentity, Constraint, SelectionBox}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::expr_ir::ScalarValue; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, Predicate, ScalarExpr}; - -fn accuracy() -> AccuracyTarget { - AccuracyTarget::max_error(0.01).failure_probability(0.01) -} - -/// The single KLL `SummaryAgg` in an explicit KLL rule fixture for `pre`. -/// Coverage semantics do not depend on the capacities visited by search. -fn kll_state(pre: Rc) -> Rc { - let dag = explicit_summary_candidate(pre, SketchAlgorithm::Kll); - let mut states: Vec<_> = OperatorNode::reachable(&dag) - .into_iter() - .filter(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::Kll) - }) - .collect(); - assert_eq!( - states.len(), - 1, - "expected one KLL candidate state: {states:#?}" - ); - states.pop().unwrap() -} - -async fn sql_state(sql: &str) -> Rc { - let catalog = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - ]), - ); - let pre = lower_sql(sql, &catalog, accuracy()) - .await - .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); - kll_state(pre) -} - -fn promql_state(query: &str) -> Rc { - kll_state(lower_promql(query, accuracy()).expect("lowering failed")) -} - -fn column(table: Option<&str>, name: &str) -> ColumnIdentity { - ColumnIdentity::new(table.map(str::to_string), name.to_string()) -} - -fn utf8(value: &str) -> ScalarValue { - ScalarValue::Utf8(value.to_string()) -} - -/// The child of the definition's `SummaryAgg`. -fn definition_child(state: &OperatorNode) -> Rc { - state.coverage().unwrap().definition().children()[0].clone() -} - -/// The definition's `SummaryAgg` carries no filter of its own. -fn assert_definition_unfiltered(state: &OperatorNode) { - let coverage = state.coverage().unwrap(); - let definition = coverage.definition(); - assert!( - matches!( - definition.asap(), - Some(ASAPOp::SummaryAgg { filter: None, .. }) - ), - "{definition:#?}" - ); -} - -/// The definition keeps the whole sub-DAG below the state: nothing moved -/// into the selection, which is the one unrestricted box. -fn assert_nothing_selected(state: &OperatorNode) { - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::default()] - ); - assert!(Rc::ptr_eq(&definition_child(state), state.children()[0])); -} - -#[tokio::test] -async fn sql_where_value_set_moves_to_selection() { - let state = - sql_state("SELECT approx_percentile_cont(latency, 0.99) FROM t WHERE region = 'us'").await; - let coverage = state.coverage().unwrap(); - assert_eq!( - coverage.selection(), - vec![SelectionBox::new( - [( - column(Some("t"), "region"), - Constraint::In(vec![utf8("us")]) - )] - .into(), - None - )] - ); - assert_definition_unfiltered(&state); - let child = definition_child(&state); - assert!( - matches!(child.non_asap(), Some(NonASAPOp::Scan { predicates, .. }) if predicates.is_empty()) - ); -} - -#[tokio::test] -async fn sql_where_range_moves_to_selection() { - let state = - sql_state("SELECT approx_percentile_cont(latency, 0.99) FROM t WHERE latency < 100").await; - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - [( - column(Some("t"), "latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Excluded(ScalarValue::Float64(100.0)), - } - )] - .into(), - None - )] - ); - assert_definition_unfiltered(&state); - let child = definition_child(&state); - assert!( - matches!(child.non_asap(), Some(NonASAPOp::Scan { predicates, .. }) if predicates.is_empty()) - ); -} - -/// The worked example without its `FILTER` clause: the rename carries -/// `region = 'us'` into the selection as `r`, and the expression condition -/// stays in the definition's scan. -#[tokio::test] -async fn sql_renamed_column_moves_and_expression_stays() { - let state = sql_state( - "SELECT job, approx_percentile_cont(latency, 0.99) \ - FROM (SELECT job, region AS r, latency FROM t \ - WHERE region = 'us' AND latency * 2 > 10) \ - GROUP BY job", - ) - .await; - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - [(column(None, "r"), Constraint::In(vec![utf8("us")]))].into(), - None - )] - ); - assert_definition_unfiltered(&state); - let project = definition_child(&state); - assert!(matches!( - project.non_asap(), - Some(NonASAPOp::Project { .. }) - )); - let scan = project.children()[0].clone(); - let Some(NonASAPOp::Scan { predicates, .. }) = scan.non_asap() else { - panic!("expected a scan, got {scan:#?}"); - }; - assert_eq!(predicates.len(), 1, "{predicates:#?}"); - assert!( - matches!(&predicates[0], Predicate(ScalarExpr::Compare { left, .. }) - if matches!(**left, ScalarExpr::Arithmetic { .. })), - "the expression condition should stay: {predicates:#?}" - ); -} - -#[test] -fn promql_offset_window_is_relative_time() { - let state = promql_state("quantile_over_time(0.99, m[1m] offset 2m)"); - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - Default::default(), - Some((Bound::Excluded(-180_000), Bound::Included(-120_000))) - )] - ); - assert!(matches!( - definition_child(&state).non_asap(), - Some(NonASAPOp::Scan { .. }) - )); -} - -/// The label matcher is below `rate`, which the walk does not pass: the -/// definition keeps all of it, the matcher included. -#[test] -fn promql_matcher_below_rate_stays_in_definition() { - let state = promql_state("quantile(0.99, rate(m{job=\"api\"}[5m]))"); - assert_nothing_selected(&state); - let scans: Vec<_> = OperatorNode::reachable(&definition_child(&state)) - .into_iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Scan { predicates, .. }) => Some(predicates.len()), - _ => None, - }) - .collect(); - assert_eq!(scans, vec![1], "the matcher stays in the scan"); -} - -/// A PromQL comparison lowers to a `Filter` above `rate`, so it picks the -/// rate outputs the state reads; the definition keeps the `rate` below it. -#[test] -fn promql_comparison_above_rate_moves_to_selection() { - let state = promql_state("quantile(0.99, rate(m[5m]) > 0)"); - let filter = state.children()[0]; - assert!(matches!(filter.non_asap(), Some(NonASAPOp::Filter { .. }))); - assert!(Rc::ptr_eq(&definition_child(&state), filter.children()[0])); - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - [( - column(None, "value"), - Constraint::Interval { - lower: Bound::Excluded(ScalarValue::Float64(0.0)), - upper: Bound::Unbounded, - }, - )] - .into(), - None - )] - ); -} - -/// A vector/vector comparison is a `BinaryOp`, which the walk does not enter. -#[test] -fn promql_vector_comparison_stays_in_definition() { - let state = promql_state("quantile(0.99, rate(a[5m]) > rate(b[5m]))"); - assert_nothing_selected(&state); - assert!(matches!( - definition_child(&state).non_asap(), - Some(NonASAPOp::BinaryOp { .. }) - )); -} - -/// A `bool` comparison lowers to a `Project` (values become 0 or 1), which -/// drops no row, so it stays in the definition. -#[test] -fn promql_bool_comparison_stays_in_definition() { - let state = promql_state("quantile(0.99, rate(m[5m]) > bool 0)"); - assert_nothing_selected(&state); - assert!(matches!( - definition_child(&state).non_asap(), - Some(NonASAPOp::Project { .. }) - )); -} diff --git a/crates/integration-tests/tests/time_range.rs b/crates/integration-tests/tests/time_range.rs index 6d7f8b8dc..acb19a473 100644 --- a/crates/integration-tests/tests/time_range.rs +++ b/crates/integration-tests/tests/time_range.rs @@ -12,8 +12,8 @@ use std::time::Duration; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::fixtures::metric_schema; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::{AggIntent, Reduction, Source}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; fn lower(q: &str) -> Rc { @@ -74,13 +74,7 @@ fn q14_increase() { fn q15_count_over_time() { assert_eq!( lower("count_over_time(http_requests_total[5m])"), - range_agg( - 300, - AggIntent::Count { - accuracy: AccuracyTarget::exact() - }, - "http_requests_total" - ), + range_agg(300, AggIntent::Count, "http_requests_total"), ); } @@ -159,11 +153,7 @@ fn q17_quantile_over_time() { lower("quantile_over_time(0.99, http_requests_total[5m])"), range_agg( 300, - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::exact() - }, + AggIntent::Quantile { col: None, q: 0.99 }, "http_requests_total", ), ); diff --git a/crates/integration-tests/tests/tumbling_window_merge.rs b/crates/integration-tests/tests/tumbling_window_merge.rs deleted file mode 100644 index 7f5c7cff2..000000000 --- a/crates/integration-tests/tests/tumbling_window_merge.rs +++ /dev/null @@ -1,887 +0,0 @@ -//! Tumbling windows merged by `SummaryMerge` answer the same per-series query -//! as one build over the whole window (#509 Example 3/4, Pattern B): tumbling windows -//! written by hand, and the planner's window-composition rule's -//! `SummaryTumblingWindowChain`, expanded into the tumbling windows the executor runs. -use crate::physical_common; - -use std::collections::BTreeMap; -use std::rc::Rc; -use std::sync::Arc; - -use asap_executor::physical_planner::test_support::views_at; -use asap_executor::physical_planner::{compile, promql_rows, CompiledPhysicalDAG}; -use asap_executor::runtime::Scope; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::values::{Batch, Value}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{explicit_local_candidates, explicit_summary_candidate}; -use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - window_composition_shared_chain_candidates, -}; -use asap_types::ir::operator::operator_properties::TimeShift; -use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains; -use asap_types::ir::physical_export::PhysicalASAPDAG; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; -use asap_types::ir::schema::Schema; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, Operator, OperatorNode, QueryRoot, -}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::test_support::compile_physical_asap_dag_with_node_ids; - -const EVALUATION_MS: i64 = 300_000; -const TUMBLING_WINDOW_WIDTH_MS: i64 = 60_000; -const TUMBLING_WINDOW_COUNT: i64 = 5; - -/// An explicit supported single-build fixture for the tumbling-window merge rules. -fn single_build(query: &str, accuracy: AccuracyTarget) -> Rc { - let root = lower_promql(query, accuracy.clone()).unwrap(); - let root = promql_rows::with_series_identity(&root).unwrap(); - if accuracy.is_exact() { - explicit_local_candidates(root) - .into_iter() - .find(|candidate| { - OperatorNode::reachable(candidate) - .iter() - .any(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - }) - .expect("an exact accumulator fixture") - } else { - // Pin DDS so the actual positive-sample interpolation and tumbling-window merge - // checks do not depend on the optimizer's preferred implementation. - explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::DDSketch) - } -} - -/// The per-entity `SummaryAgg` of `root` over its raw `TimeRange`. -fn summary_agg(root: &Rc) -> Rc { - OperatorNode::reachable(root) - .into_iter() - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("a summary build") -} - -/// Rewrite `root` so its summary is built in tumbling windows: tumbling window `i` is -/// `TimeRange(1m)` over `TimeShift(i·1m)` over the same `Scan`, and a -/// `SummaryMerge` combines the tumbling windows. Returns the root and the tumbling windows. -fn tumbling_windows(root: &Rc) -> (Rc, Vec>) { - let agg = summary_agg(root); - let Some(NonASAPOp::TimeRange { kind, child, .. }) = agg.children()[0].non_asap() else { - panic!("expected a raw time range under the summary"); - }; - let tumbling_windows: Vec<_> = (0..TUMBLING_WINDOW_COUNT) - .map(|i| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: i * TUMBLING_WINDOW_WIDTH_MS, - at: None, - }, - child: child.clone(), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(TUMBLING_WINDOW_WIDTH_MS as u64), - kind: *kind, - child: shifted, - })) - .unwrap(); - OperatorNode::new_shared(agg.operator.map_children(|_| range.clone())).unwrap() - }) - .collect(); - let merged = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: tumbling_windows.clone(), - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(); - let root = replace(root, &agg, &merged); - root.validate_structure().unwrap(); - (root, tumbling_windows) -} - -/// `root` with its summary built in tumbling windows by the planner's -/// window-composition rule, for a query repeating every minute: the -/// planned root (a `SummaryMerge` over one `SummaryTumblingWindowChain`) and its expansion -/// into the tumbling windows the executor runs. -fn planned_tumbling_windows(root: &Rc) -> (Rc, Rc) { - let roots = [QueryRoot::Operator(Rc::clone(root))]; - let rule_candidates = - tumbling_window_candidates(&roots, &[Some(TUMBLING_WINDOW_WIDTH_MS as u64)]); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one build to split: {rule_candidates:?}") - }; - assert_eq!( - ( - rule_candidate.tumbling_window_width_ms, - rule_candidate.tumbling_window_count - ), - (TUMBLING_WINDOW_WIDTH_MS as u64, 5) - ); - let mut pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let pass2_candidate = pass2_candidates.pop().unwrap(); - assert!(pass2_candidates.is_empty()); - let QueryRoot::Operator(planned) = pass2_candidate.roots.unwrap().remove(0) else { - panic!("an operator root") - }; - planned.validate_structure().unwrap(); - assert_eq!( - planned.guarantee, root.guarantee, - "merged tumbling windows keep the whole build's guarantee" - ); - let expanded = expand_tumbling_window_chains(&planned).unwrap(); - expanded.validate_structure().unwrap(); - (planned, expanded) -} - -/// `node` with `from` replaced by `to`. -fn replace( - node: &Rc, - from: &Rc, - to: &Rc, -) -> Rc { - if Rc::ptr_eq(node, from) { - return to.clone(); - } - Rc::new( - node.with_new_children(|child| replace(child, from, to)) - .unwrap() - .with_guarantee(node.guarantee.clone()), - ) -} - -/// One raw data sample: series, timestamp (ms) and value. -type RawDataSample = (&'static str, i64, f64); - -/// Two series, four positive samples per minute in [0, 5m), each -/// `phase_ms` after its 15-s step (0: on the tumbling-window boundaries). -fn samples(phase_ms: i64) -> Vec { - let mut rows = vec![]; - for (s, series) in ["a", "b"].into_iter().enumerate() { - for minute in 0..TUMBLING_WINDOW_COUNT { - for j in 0..4 { - let ts = minute * TUMBLING_WINDOW_WIDTH_MS + j * 15_000 + phase_ms; - rows.push((series, ts, (s * 100) as f64 + (minute * 4 + j + 1) as f64)); - } - } - } - rows -} - -struct Exported { - dag: PhysicalASAPDAG, - /// Each `SummaryAgg` node with its tumbling window. - builds: Vec<(u64, i64)>, -} - -fn export(root: &Rc, assignment: &MaterializationAssignment) -> Exported { - let timed = - apply_materialization_timings(root, assignment, &mut NodeToTimedNodeCache::new()).unwrap(); - let compiled = compile_physical_asap_dag_with_node_ids(&timed).unwrap(); - let tumbling_window = |node: &OperatorNode| -> i64 { - match node.children()[0].non_asap() { - Some(NonASAPOp::TimeShift { shift, .. }) => shift.offset_ms / TUMBLING_WINDOW_WIDTH_MS, - _ => 0, - } - }; - let mut builds = vec![]; - for node in &compiled.dag.nodes { - let logical = compiled.node_ids.operator_node(node.id).unwrap(); - if let Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) = &logical.operator { - builds.push((node.id as u64, tumbling_window(child))); - } - } - Exported { - dag: compiled.dag, - builds, - } -} - -fn schema_of(dag: &PhysicalASAPDAG, id: u64) -> Arc { - Arc::new( - dag.nodes - .iter() - .find(|node| node.id as u64 == id) - .unwrap() - .output_schema - .clone(), - ) -} - -/// A store holding every sample as raw data at the `Scan` of `m`; each -/// `TimeRange` keeps its own window of them. -fn raw_store(dag: &PhysicalASAPDAG, samples: &[RawDataSample]) -> InMemoryDataStore { - physical_common::raw_data_store(dag, |_, schema| { - samples - .iter() - .map(|&(series, ts, value)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ("series".to_string(), series.to_string()), - ]); - promql_rows::series_row(schema, &labels, ts, value).unwrap() - }) - .collect() - }) -} - -fn run(plan: &CompiledPhysicalDAG, store: InMemoryDataStore, scope: Scope) -> Vec> { - let mut rows: Vec> = physical_common::execute(plan, store, scope) - .remove(0) - .iter() - .flat_map(|batch| batch.to_rows().to_vec()) - .collect(); - rows.sort_by_key(|row| format!("{row:?}")); - rows -} - -/// `Value` has no equality; its debug form identifies every scalar exactly. -fn render(rows: &[Vec]) -> Vec { - rows.iter().map(|row| format!("{row:?}")).collect() -} - -fn has_timestamp(row: &[Value], ms: i64) -> bool { - row.iter() - .any(|v| matches!(v, Value::Timestamp(t) if *t == ms)) -} - -fn query_scope() -> Scope { - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - } -} - -/// Compile `root` with everything at query time (Pattern B2: tumbling windows rebuilt -/// from raw rows at query time). -fn compile_at_query_time( - root: &Rc, -) -> Result<(Exported, CompiledPhysicalDAG), String> { - let exported = export(root, &MaterializationAssignment::default()); - let plan = - compile(&exported.dag, &[], &[exported.dag.roots[0] as u64]).map_err(|e| e.to_string())?; - Ok((exported, plan)) -} - -fn execute(root: &Rc, samples: &[RawDataSample]) -> Vec> { - let (exported, plan) = compile_at_query_time(root).unwrap(); - run(&plan, raw_store(&exported.dag, samples), query_scope()) -} - -/// Build the tumbling windows in ingestion runs (stored `MatAtIngestion`, each with its -/// own build timestamp), then merge them in the query run (Pattern B1). -fn execute_retained( - root: &Rc, - tumbling_windows: &[Rc], - samples: &[RawDataSample], -) -> Vec> { - let mut assignment = MaterializationAssignment::default(); - for tumbling_window in tumbling_windows { - assignment.set( - tumbling_window, - Materialization::MatAtIngestion(Medium::Memory), - ); - } - execute_retained_as(root, &assignment, samples) -} - -/// The planner's tumbling-window chain maintained at ingestion time, as Stage 2's B1 -/// option assigns it (`MatAtIngestion` on the chain), then expanded: each -/// tumbling window carries the chain's timing and materialization, so the tumbling windows are -/// built in ingestion runs and merged in the query run. -fn execute_maintained_chain( - planned: &Rc, - samples: &[RawDataSample], -) -> Vec> { - let chain = OperatorNode::reachable(planned) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .expect("a tumbling-window chain"); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&chain, Materialization::MatAtIngestion(Medium::Memory)); - let timed = - apply_materialization_timings(planned, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let expanded = expand_tumbling_window_chains(&timed).unwrap(); - for tumbling_window in OperatorNode::reachable(&expanded) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - { - assert_eq!( - tumbling_window.materialization, - Materialization::MatAtIngestion(Medium::Memory) - ); - } - execute_retained_as(&expanded, &MaterializationAssignment::default(), samples) -} - -fn execute_retained_as( - root: &Rc, - assignment: &MaterializationAssignment, - samples: &[RawDataSample], -) -> Vec> { - let mut exported = export(root, assignment); - let builds: Vec = exported.builds.iter().map(|&(build, _)| build).collect(); - let views = views_at(&mut exported.dag, &builds).unwrap(); - // Each tumbling window's ingestion run refreshes its view from the raw samples. - let mut store = InMemoryDataStore::default(); - for &(build, tumbling_window) in &exported.builds { - let plan = compile(&exported.dag, &[], &[build]).unwrap(); - let end = EVALUATION_MS - tumbling_window * TUMBLING_WINDOW_WIDTH_MS; - let rows = run( - &plan, - raw_store(&exported.dag, samples), - Scope::Ingestion { - window_start_ms: end - TUMBLING_WINDOW_WIDTH_MS, - window_end_ms: end, - }, - ); - assert!(rows.iter().all(|row| has_timestamp(row, end))); - store = store - .with_materialized_view( - build as usize, - vec![Batch::try_new(schema_of(&exported.dag, build), rows).unwrap()], - ) - .unwrap(); - } - let plan = compile(&exported.dag, &views, &[exported.dag.roots[0] as u64]).unwrap(); - run(&plan, store, query_scope()) -} - -fn values(rows: &[Vec]) -> Vec { - rows.iter() - .flat_map(|row| { - row.iter().filter_map(|v| match v { - Value::Float64(v) => Some(*v), - _ => None, - }) - }) - .collect() -} - -/// Returns the per-series values, which every execution path agrees on. -fn assert_tumbling_window_merge_matches_single_build( - query: &str, - accuracy: AccuracyTarget, - samples: &[RawDataSample], -) -> Vec { - let single = single_build(query, accuracy); - let (tumbling_windows, tumbling_window_builds) = tumbling_windows(&single); - let expected = execute(&single, samples); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert!(expected.iter().all(|row| has_timestamp(row, EVALUATION_MS))); - let result = values(&expected); - let expected = render(&expected); - assert_eq!( - render(&execute(&tumbling_windows, samples)), - expected, - "tumbling windows rebuilt at query time" - ); - assert_eq!( - render(&execute_retained( - &tumbling_windows, - &tumbling_window_builds, - samples - )), - expected, - "retained tumbling windows" - ); - let (planned, expanded) = planned_tumbling_windows(&single); - assert_eq!( - render(&execute(&expanded, samples)), - expected, - "the planner's tumbling-window chain, expanded" - ); - assert_eq!( - render(&execute_maintained_chain(&planned, samples)), - expected, - "the planner's tumbling-window chain, maintained at ingestion time" - ); - let unexpanded = compile_at_query_time(&planned).err().unwrap_or_default(); - assert!( - unexpanded.contains("SummaryTumblingWindowChain"), - "{unexpanded}" - ); - result -} - -/// Exact per-series Sum over 5 merged 1-minute tumbling windows equals one 5-minute -/// build, timestamped at the evaluation time. -#[test] -fn exact_sum_over_tumbling_windows_matches_single_build() { - let sums = assert_tumbling_window_merge_matches_single_build( - "sum_over_time(m[5m])", - AccuracyTarget::exact(), - // 1 ms past each step: the window `(0, 5m]` holds all 20 samples. - &samples(1), - ); - assert_eq!(sums, vec![210., 2210.]); -} - -/// The selected quantile sketch over 5 merged 1-minute tumbling windows equals one -/// 5-minute build, within the target of the exact p99 (20 and 120). -#[test] -fn quantile_over_tumbling_windows_matches_single_build() { - let p99 = assert_tumbling_window_merge_matches_single_build( - "quantile_over_time(0.99, m[5m])", - AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative), - &samples(0), - ); - let mut p99 = p99; - p99.sort_by(f64::total_cmp); - assert_eq!(p99.len(), 2); - for (estimate, exact) in p99.into_iter().zip([20., 120.]) { - assert!( - (estimate - exact).abs() <= 0.05 * exact, - "{estimate} vs {exact}" - ); - } -} - -/// Exact counter Rate over 5 merged 1-minute tumbling windows equals one 5-minute -/// build: the merged increase is divided by the whole 5-minute window, not -/// by one tumbling window's minute. -#[test] -fn exact_rate_over_tumbling_windows_matches_single_build() { - let rates = assert_tumbling_window_merge_matches_single_build( - "rate(m[5m])", - AccuracyTarget::exact(), - &samples(0), - ); - assert_eq!(rates.len(), 2); - // 19 counted over 285 s of samples, extrapolated to the 5-min window. - for rate in rates { - assert!((rate - 20.0 / 300.0).abs() < 1e-9, "{rate}"); - } -} - -/// With samples off the tumbling-window boundaries, Increase extrapolates to the -/// start of the whole 5-minute window, as one build does, not to the start -/// of the newest tumbling window. -#[test] -fn exact_increase_off_the_tumbling_window_boundaries_matches_single_build() { - let increases = assert_tumbling_window_merge_matches_single_build( - "increase(m[5m])", - AccuracyTarget::exact(), - &samples(7_000), - ); - assert_eq!(increases.len(), 2); -} - -/// The single build of `query`, evaluated at `evaluation_ms` over every -/// sample, as values. -fn whole_build_values_at(query: &str, evaluation_ms: i64, samples: &[RawDataSample]) -> Vec { - let root = single_build(query, AccuracyTarget::exact()); - let (exported, plan) = compile_at_query_time(&root).unwrap(); - values(&run( - &plan, - raw_store(&exported.dag, samples), - Scope::Query { - evaluation_time_ms: evaluation_ms, - }, - )) -} - -/// A counter over `[5m] offset 1m` evaluated a minute later reads the same -/// window as one without the offset: the counter window ends at the -/// evaluation time minus the offset, not at the evaluation time. -#[test] -fn offset_counter_window_ends_at_the_offset() { - let samples = samples(7_000); - for function in ["rate", "increase"] { - let plain = whole_build_values_at(&format!("{function}(m[5m])"), EVALUATION_MS, &samples); - let shifted = whole_build_values_at( - &format!("{function}(m[5m] offset 1m)"), - EVALUATION_MS + TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_eq!(shifted, plain, "{function}"); - } -} - -/// A counter over `[5m] @ 300` reads the window ending at 300 s whatever the -/// evaluation time, and `@ 300 offset 1m` the one ending at 240 s: the same -/// values as the unanchored counter evaluated at 300 s and at 240 s. Today -/// it reads the window ending at the evaluation time (#748). -#[test] -#[ignore = "the counter window drops an @ anchor and its offset: #748"] -fn anchored_counter_window_ends_at_the_anchor() { - let samples = samples(7_000); - for function in ["rate", "increase"] { - let at_300 = whole_build_values_at(&format!("{function}(m[5m])"), EVALUATION_MS, &samples); - let at_240 = whole_build_values_at( - &format!("{function}(m[5m])"), - EVALUATION_MS - TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_ne!(at_300, at_240, "{function}: the two windows differ"); - let anchored = whole_build_values_at( - &format!("{function}(m[5m] @ 300)"), - EVALUATION_MS + TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_eq!(anchored, at_300, "{function} @ 300"); - let anchored_offset = whole_build_values_at( - &format!("{function}(m[5m] @ 300 offset 1m)"), - EVALUATION_MS + TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_eq!(anchored_offset, at_240, "{function} @ 300 offset 1m"); - } -} - -/// Tumbling windows whose ranges read the scan through two stacked `TimeShift`s (the -/// tumbling window's shift over a zero shift) compile and answer as the single build: -/// the per-entity build looks through every shift to its raw source. -#[test] -fn tumbling_windows_over_stacked_time_shifts_match_single_build() { - let single = single_build("increase(m[5m])", AccuracyTarget::exact()); - let (tumbling_windows, _) = tumbling_windows(&single); - let agg = summary_agg(&single); - let Some(NonASAPOp::TimeRange { child: scan, .. }) = agg.children()[0].non_asap() else { - panic!("expected a raw time range under the summary"); - }; - let zero_shift = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 0, - at: None, - }, - child: scan.clone(), - })) - .unwrap(); - let stacked = replace(&tumbling_windows, scan, &zero_shift); - let expected = render(&execute(&single, &samples(0))); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert_eq!(render(&execute(&stacked, &samples(0))), expected); -} - -/// Exact counter Increase over 5 merged 1-minute tumbling windows equals one 5-minute -/// build: the merge joins consecutive tumbling windows across their boundaries. -#[test] -fn exact_increase_over_tumbling_windows_matches_single_build() { - let increases = assert_tumbling_window_merge_matches_single_build( - "increase(m[5m])", - AccuracyTarget::exact(), - &samples(0), - ); - assert_eq!(increases.len(), 2); -} - -/// A middle tumbling window with no samples, for one series or for every series: the -/// merge joins the tumbling windows on either side of the empty one, and Increase -/// matches one 5-minute build. -#[test] -fn exact_increase_over_an_empty_middle_tumbling_window_matches_single_build() { - let middle = 2 * TUMBLING_WINDOW_WIDTH_MS..3 * TUMBLING_WINDOW_WIDTH_MS; - for emptied in [&["a"][..], &["a", "b"]] { - let gapped: Vec = samples(0) - .into_iter() - .filter(|(series, ts, _)| !(emptied.contains(series) && middle.contains(ts))) - .collect(); - let increases = assert_tumbling_window_merge_matches_single_build( - "increase(m[5m])", - AccuracyTarget::exact(), - &gapped, - ); - assert_eq!(increases.len(), 2, "{emptied:?}"); - } -} - -/// A counter over `[5m] offset 1m` split into the planner's tumbling windows, read -/// at query time or maintained at ingestion time, matches one build: the -/// tumbling windows' shifts fold the outer offset, and the merged counter window ends -/// at the evaluation time minus the offset. -#[test] -fn a_counter_over_tumbling_windows_with_an_outer_offset_matches_single_build() { - let samples = samples(7_000); - for function in ["rate", "increase"] { - let single = single_build( - &format!("{function}(m[5m] offset 1m)"), - AccuracyTarget::exact(), - ); - let expected = render(&execute(&single, &samples)); - assert_eq!(expected.len(), 2, "{function}: {expected:?}"); - let (planned, expanded) = planned_tumbling_windows(&single); - assert_eq!( - render(&execute(&expanded, &samples)), - expected, - "{function}: expanded tumbling windows" - ); - assert_eq!( - render(&execute_maintained_chain(&planned, &samples)), - expected, - "{function}: tumbling windows maintained at ingestion time" - ); - } -} - -/// `root`'s summary built from tumbling windows `(shift, width)` merged by one -/// `SummaryMerge`, both in minutes, or the IR's refusal of that merge. -fn tumbling_windows_at( - root: &Rc, - tumbling_windows: &[(i64, i64)], -) -> Result, String> { - let agg = summary_agg(root); - let Some(NonASAPOp::TimeRange { kind, child, .. }) = agg.children()[0].non_asap() else { - panic!("expected a raw time range under the summary"); - }; - let children = tumbling_windows - .iter() - .map(|&(shift, width)| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift * TUMBLING_WINDOW_WIDTH_MS, - at: None, - }, - child: child.clone(), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis((width * TUMBLING_WINDOW_WIDTH_MS) as u64), - kind: *kind, - child: shifted, - })) - .unwrap(); - OperatorNode::new_shared(agg.operator.map_children(|_| range.clone())).unwrap() - }) - .collect(); - let merged = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: None, - group_by: None, - })) - .map_err(|error| format!("{error:?}"))?; - Ok(replace(root, &agg, &merged)) -} - -/// Counter tumbling windows form one window only back to back: with a gap between -/// them, compiling the counter evaluation is an error; overlapping tumbling windows -/// are already refused by the `SummaryMerge`, whose counter inputs must be -/// disjoint in time. -#[test] -fn counter_tumbling_windows_with_a_gap_or_an_overlap_are_rejected() { - let single = single_build("rate(m[5m])", AccuracyTarget::exact()); - let gapped = tumbling_windows_at(&single, &[(0, 1), (2, 1)]).unwrap(); - let error = compile_at_query_time(&gapped).err().unwrap_or_default(); - assert!(error.contains("ambiguous logical windows"), "{error}"); - let overlap = tumbling_windows_at(&single, &[(0, 2), (1, 2)]) - .err() - .unwrap_or_default(); - assert!(overlap.contains("InputsOverlap"), "{overlap}"); -} - -/// A 2-min and a 5-min quantile, both every minute, on 1-min tumbling windows: the -/// window-composition rule offers one 5-tumbling-window chain both read, the 2-min -/// query merging only its newest 2 tumbling windows. Expanded, rebuilt at query time -/// or maintained at ingestion time, each query's answer from the shared -/// chain equals its answer from its own chain. -#[test] -fn newest_tumbling_windows_of_a_shared_chain_match_a_separate_chain() { - let accuracy = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let roots = [ - QueryRoot::Operator(single_build( - "quantile_over_time(0.5, m[2m])", - accuracy.clone(), - )), - QueryRoot::Operator(single_build("quantile_over_time(0.99, m[5m])", accuracy)), - ]; - let grid = Some(TUMBLING_WINDOW_WIDTH_MS as u64); - let rule_candidates = tumbling_window_candidates(&roots, &[grid, grid]); - assert_eq!(rule_candidates.len(), 2, "{rule_candidates:?}"); - let pass2_candidates = window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let both = |shared: bool| -> Vec> { - pass2_candidates - .iter() - .find(|c| { - c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty() != shared - }) - .expect("both builds in tumbling windows") - .roots - .as_ref() - .unwrap() - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => Rc::clone(node), - QueryRoot::Scalar(_) => panic!("an operator root"), - }) - .collect() - }; - let (shared, separate) = (both(true), both(false)); - let newest = OperatorNode::reachable(&shared[0]) - .into_iter() - .find_map(|n| match n.asap() { - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - }) => Some(*newest_tumbling_windows), - _ => None, - }); - assert_eq!(newest, Some(Some(2))); - let data = samples(0); - for (query, (shared, separate)) in shared.iter().zip(&separate).enumerate() { - let expected = execute(&expand_tumbling_window_chains(separate).unwrap(), &data); - assert!(!expected.is_empty()); - assert_eq!( - render(&execute( - &expand_tumbling_window_chains(shared).unwrap(), - &data - )), - render(&expected), - "Q{}: the shared chain rebuilt at query time", - query + 1 - ); - assert_eq!( - render(&execute_maintained_chain(shared, &data)), - render(&expected), - "Q{}: the shared chain maintained at ingestion time", - query + 1 - ); - } -} - -/// Runs `root` at query time at `evaluation_ms` over every sample; each -/// `TimeRange` keeps its own window `(T - s - w, T - s]` for a range of -/// width `w` over a shift of `s` (none for an unshifted range). -fn execute_ranges_at_their_windows( - root: &Rc, - samples: &[RawDataSample], - evaluation_ms: i64, -) -> Vec> { - let (exported, plan) = compile_at_query_time(root).unwrap(); - let scope = Scope::Query { - evaluation_time_ms: evaluation_ms, - }; - run(&plan, raw_store(&exported.dag, samples), scope) -} - -/// A 5-min quantile every 30 s (10 × 30-s tumbling windows) and a 5-min quantile -/// every minute (5 × 1-min tumbling windows): the rule offers one chain of 30-s -/// tumbling windows both read in full, each at its own interval. Expanded, each -/// query's answer from the shared chain equals one build over its window, -/// at 5 min, and for the 30-s query also at 4.5 min, off the 1-min grid. -#[test] -fn queries_at_different_intervals_read_one_chain_of_the_finer_tumbling_windows() { - let accuracy = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let singles = [ - single_build("quantile_over_time(0.5, m[5m])", accuracy.clone()), - single_build("quantile_over_time(0.99, m[5m])", accuracy), - ]; - let roots: Vec = singles - .iter() - .map(|root| QueryRoot::Operator(Rc::clone(root))) - .collect(); - let rule_candidates = tumbling_window_candidates( - &roots, - &[Some(30_000), Some(TUMBLING_WINDOW_WIDTH_MS as u64)], - ); - let widths: Vec = rule_candidates - .iter() - .map(|c| c.tumbling_window_width_ms) - .collect(); - assert_eq!(widths, [30_000, TUMBLING_WINDOW_WIDTH_MS as u64]); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && !c.shared_chains.is_empty()) - .expect("one shared chain"); - // The whole set's shared chains, built alone, are the same candidates. - let alone = window_composition_shared_chain_candidates(&roots, &rule_candidates); - let first = alone.first().expect("one shared chain alone"); - assert_eq!(first.shared_chains, shared.shared_chains); - assert_eq!(first.roots.as_ref().ok(), shared.roots.as_ref().ok()); - assert_eq!( - ( - shared.shared_chains[0].tumbling_window_width_ms, - shared.shared_chains[0].tumbling_window_count - ), - (30_000, 10) - ); - let data = samples(0); - let shared_roots: Vec> = shared - .roots - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => expand_tumbling_window_chains(&node).unwrap(), - QueryRoot::Scalar(_) => panic!("an operator root"), - }) - .collect(); - for (query, evaluation_ms) in [(0, EVALUATION_MS), (1, EVALUATION_MS), (0, 270_000)] { - let expected = execute_ranges_at_their_windows(&singles[query], &data, evaluation_ms); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert_eq!( - render(&execute_ranges_at_their_windows( - &shared_roots[query], - &data, - evaluation_ms - )), - render(&expected), - "Q{} at {evaluation_ms} ms", - query + 1 - ); - } -} - -/// A 5-min quantile every minute (5 × 1-min tumbling windows) and a 3-min one every -/// 90 s (2 × 90-s tumbling windows): neither width divides the other, so the rule -/// offers one chain at their gcd, 10 × 30-s tumbling windows, the 3-min query merging -/// the newest 6. Expanded, each query's answer from the shared chain equals -/// one build over its window, at times on each query's own grid. -#[test] -fn queries_at_60_and_90_seconds_read_one_chain_of_30_second_tumbling_windows() { - let accuracy = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let singles = [ - single_build("quantile_over_time(0.5, m[5m])", accuracy.clone()), - single_build("quantile_over_time(0.99, m[3m])", accuracy), - ]; - let roots: Vec = singles - .iter() - .map(|root| QueryRoot::Operator(Rc::clone(root))) - .collect(); - let rule_candidates = tumbling_window_candidates( - &roots, - &[Some(TUMBLING_WINDOW_WIDTH_MS as u64), Some(90_000)], - ); - let widths: Vec<(u64, u32)> = rule_candidates - .iter() - .map(|c| (c.tumbling_window_width_ms, c.tumbling_window_count)) - .collect(); - assert_eq!(widths, [(TUMBLING_WINDOW_WIDTH_MS as u64, 5), (90_000, 2)]); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && !c.shared_chains.is_empty()) - .expect("one shared chain at the gcd"); - assert_eq!( - ( - shared.shared_chains[0].tumbling_window_width_ms, - shared.shared_chains[0].tumbling_window_count - ), - (30_000, 10) - ); - let data = samples(0); - let shared_roots: Vec> = shared - .roots - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => expand_tumbling_window_chains(&node).unwrap(), - QueryRoot::Scalar(_) => panic!("an operator root"), - }) - .collect(); - for (query, evaluation_ms) in [(0, EVALUATION_MS), (1, 270_000), (1, 180_000)] { - let expected = execute_ranges_at_their_windows(&singles[query], &data, evaluation_ms); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert_eq!( - render(&execute_ranges_at_their_windows( - &shared_roots[query], - &data, - evaluation_ms - )), - render(&expected), - "Q{} at {evaluation_ms} ms", - query + 1 - ); - } -} diff --git a/crates/logical-asap-optimization/Cargo.toml b/crates/logical-asap-optimization/Cargo.toml index 665246936..3154d7233 100644 --- a/crates/logical-asap-optimization/Cargo.toml +++ b/crates/logical-asap-optimization/Cargo.toml @@ -21,7 +21,7 @@ asap-frontend-promql = { path = "../frontend-promql", optional = true } [features] # The test helpers (`test_support`) the later stage crates' tests share. -test-support = ["dep:asap-frontend-promql"] +test-support = ["dep:asap-frontend-promql", "asap-types/test-support"] [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index 3864772dc..49b0a8791 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -4,7 +4,7 @@ //! (`asap_types::stages::LogicalASAPMemoCandidates`) that Stage 2 implements. //! //! Identical-sub-DAG sharing over the logical DAG (Stage 0, the `stage0` -//! module, behind [`generate_dag_of_logical_memo_groups`]) runs before the +//! module, behind [`memo_of_queries`]) runs before the //! stages; Stage 1 adds the ASAP-aware sharing that is invisible at the //! logical level (Pass 2, the `asap_aware_cse` module), e.g. //! `Quantile(x, 0.99)` and `Quantile(x, 0.95)` reading one built sketch. @@ -13,8 +13,7 @@ //! (`asap-accuracy`). It depends on no other stage crate, and no other //! stage crate depends on it except in their tests (CI checks the crate //! graph): the stage-boundary types (its input -//! `DagOfLogicalMemoGroups`, Pass 1's `PerComputationLogicalMemoCandidates` -//! and Pass 2's `LogicalASAPMemoCandidates`) are in `asap_types::stages`, and +//! `Memo`, Pass 1's `Memo` and Pass 2's `LogicalASAPMemoCandidates`) are in `asap_types::stages`, and //! the planner facade calls each stage in turn. It depends on no //! query-language frontend either, except with the `test-support` feature, //! whose test helpers (`test_support`) lower PromQL test queries through @@ -33,35 +32,29 @@ //! re-exports the internals other crates' tests call. //! //! - `stage0` — Stage 0: hash-consing the workload's logical DAG into memo -//! groups (`stage0::cse`), behind [`generate_dag_of_logical_memo_groups`]. -//! - `pass1::per_computation_candidates` — Pass 1: per-computation candidate +//! groups through `Memo::find_or_add_group`, behind [`memo_of_queries`]. +//! - `pass1::computation_targets` — Pass 1: per-computation candidate //! generation and composition, behind -//! [`generate_per_computation_logical_memo_candidates`]. -//! - `pass2::asap_aware_cse` and `pass2::memo` — Pass 2: ASAP-aware -//! common-subexpression elimination and the memo rules, behind +//! [`pass1_memo`]. +//! - `pass2::memo` — Pass 2: the rules over groups, behind //! [`generate_logical_asap_memo_candidates`]. //! - `pass1::realization` — the summary families that can serve an aggregate //! intent, the intent's accuracy target and `(ε, δ)` budget, and default //! sketch sizing, shared by Pass 1 and Pass 2. -//! - `deployment_capabilities` — the deployment's supported summaries -//! ([`DeploymentCapabilities`]). //! //! Accuracy guarantees come from the built-in analytical accuracy rules in //! `asap-accuracy` (per-family estimators, composition and budget //! allocation). They derive a machine-readable -//! [`AccuracyGuarantee`](asap_types::ir::properties::AccuracyGuarantee) +//! [`AccuracyGuarantee`](asap_types::accuracy::AccuracyGuarantee) //! for every finalized value and propagate it through //! approximate-over-approximate compositions under conservative rules (no //! independence assumptions, unknown statistics stay unknown). A //! composition with no rule leaves the guarantee `None` and keeps the -//! candidate. Pass 1 takes the deployment's `AccuracyModel` and evidence -//! only to prune summaries no query reading them can use (when they cannot -//! accept an output without its guarantee) and sizes every summary for its -//! readers; plan selection (Stage 3) checks each selected candidate's -//! guarantee against its query's `AccuracyTarget` and never resizes. +//! candidate. Pass 1 prunes nothing for accuracy; Pass 2 gives each readout +//! its required accuracy and eliminates the candidates no query can use, +//! with the deployment's measured accuracy (`AccuracyEvidence`) read before +//! each summary's theoretical guarantee. -/// The deployment's capabilities: supported summaries and what it retains. -mod deployment_capabilities; mod pass1; mod pass2; /// Stage 0: the memo groups of the workload's query roots. @@ -75,28 +68,15 @@ pub mod test_support; // developer tools. Every other item is crate-private; the ones other crates' // tests call are re-exported by `test_support`. -pub use deployment_capabilities::{ - DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, -}; -// Stage 0: the memo groups of the workload's query roots -// (`asap_types::stages::DagOfLogicalMemoGroups`). -pub use stage0::generate_dag_of_logical_memo_groups; +// Stage 0: the memo groups of the workload's query roots, without +// candidates (`asap_types::stages::Memo`). +pub use stage0::memo_of_queries; // Pass 1: its entry and its errors. Its output and the types the output // holds are in `asap_types::stages`. -pub use pass1::{ - generate_per_computation_logical_memo_candidates, PerComputationCandidateError, - PerComputationLogicalMemoCandidatesError, -}; +pub use pass1::{pass1_memo, Pass1Error, PerComputationCandidateError}; // Pass 2: its entry and its error. -pub use pass1::memo_population::CandidatePopulationError; -pub use pass2::logical_memo_candidates::{ +pub use pass2::memo::rules::Pass2RuleError; +pub use pass2::{ generate_logical_asap_memo_candidates, pass2_after_local_window_composition, pass2_local_window_composition, }; -pub use pass2::memo::state_groups::Pass2RuleError; -// The CandidateLogicalASAPDAGs a memo stands for (developer tools' -// reference pricing). -pub use pass2::memo::enumeration::{ - build_logical_asap_dags_from_chosen_candidates, enumerate, AllChosenCandidatesPerMemoGroup, - ChosenCandidatePerMemoGroup, ErrorBuildingLogicalASAPDagsFromChosenCandidates, -}; diff --git a/crates/logical-asap-optimization/src/pass1.rs b/crates/logical-asap-optimization/src/pass1.rs index 2f115053b..8bbd209c1 100644 --- a/crates/logical-asap-optimization/src/pass1.rs +++ b/crates/logical-asap-optimization/src/pass1.rs @@ -1,26 +1,49 @@ -//! Stage 1, Pass 1: per-computation candidate generation and composition, -//! and the summary realizations it offers. Its one entry is -//! [`generate_per_computation_logical_memo_candidates`], whose output -//! ([`PerComputationLogicalMemoCandidates`](asap_types::stages::PerComputationLogicalMemoCandidates)) -//! Pass 2 extends. +//! Stage 1, Pass 1: per-computation candidate generation and the summary +//! realizations it offers. Its one entry is +//! [`pass1_memo`], which fills each group with its initial candidates (exact, +//! exact accumulator, one summary build per listed parameter configuration, +//! fused builds), with every intermediate result its own group (a [`Memo`]), +//! which Pass 2 extends. -/// Stage 1, Pass 1's entry and output: the per-computation candidates on -/// the memo groups ([`PerComputationLogicalMemoCandidates`](asap_types::stages::PerComputationLogicalMemoCandidates)). -pub(crate) mod logical_memo_candidates; -/// Pass 1's candidates on Stage 0's memo groups (`try_populate_pass1`). -pub(crate) mod memo_population; -/// Stage 1, Pass 1's computation targets, their candidates and their -/// composition. No execution timing is assigned: that is a Stage 2 +/// Stage 1, Pass 1's computation targets. No execution timing is assigned: that is a Stage 2 /// materialization decision. Its target list /// (`PerComputationCandidates`) is Pass 1's working form, public only to /// tests (`crate::test_support`, feature `test-support`). -pub(crate) mod per_computation_candidates; +pub(crate) mod computation_targets; /// Frontend-to-Pass-1 acceptance tests over Pass 1's working form. #[cfg(test)] -mod per_computation_candidates_tests; +mod computation_targets_tests; pub(crate) mod realization; -pub use logical_memo_candidates::{ - generate_per_computation_logical_memo_candidates, PerComputationLogicalMemoCandidatesError, -}; -pub use per_computation_candidates::PerComputationCandidateError; +use asap_types::deployment_capabilities::SupportedSummary; +use asap_types::stages::Memo; +use computation_targets::generate_per_computation_candidates; +pub use computation_targets::PerComputationCandidateError; + +/// Why [`pass1_memo`] failed. +#[derive(Debug, thiserror::Error)] +pub enum Pass1Error { + /// A query's DAG cannot be planned: it already holds ASAP or physical + /// annotations, or a measure's accuracy target is invalid. + #[error(transparent)] + PerComputation(#[from] PerComputationCandidateError), +} + +/// Stage 1, Pass 1: the initial candidates of the memo groups of `memo` +/// (Stage 0's, without candidates). It finds one computation target per +/// measure of each aggregate group with a realization besides exact +/// execution, drops the summaries the deployment does not support +/// (`supported_summaries`, the deployment's +/// `DeploymentCapabilities::supported_summaries`; `None` supports every +/// one), and adds each way to compute a group as candidates, through +/// `Memo::find_or_add_group` and `Memo::add_candidate`. Accuracy prunes +/// nothing here: Pass 2's dead-candidate elimination does. +pub fn pass1_memo( + memo: Memo, + supported_summaries: Option<&[SupportedSummary]>, +) -> Result { + let per_computation = generate_per_computation_candidates(&memo)? + .without_unsupported_summaries(supported_summaries); + let _ = (memo, per_computation); + todo!("#1207 PR 3: add each way to compute as candidates, one per listed θ") +} diff --git a/crates/logical-asap-optimization/src/pass1/computation_targets.rs b/crates/logical-asap-optimization/src/pass1/computation_targets.rs new file mode 100644 index 000000000..3aac87266 --- /dev/null +++ b/crates/logical-asap-optimization/src/pass1/computation_targets.rs @@ -0,0 +1,1685 @@ +//! Stage 1 (logical ASAP-aware optimization), Pass 1: per-computation +//! candidate generation, as defined in +//! `docs/design_docs/proposals/planner-layering.md`. +//! +//! [`generate_per_computation_candidates`] walks the memo groups of the +//! workload's memo groups ([`Memo`]) and +//! returns, for every eligible computation, its unranked candidates. A +//! computation ([`ComputationTarget`]) is one measure of an `Aggregate` +//! group whose intent has a realization besides plain exact execution, once +//! however many queries read the group; nested aggregates and operators +//! read by scalar roots are found too. +//! Pass 1 does not consider sharing across computations; that is Pass 2. +//! +//! - A candidate says how a target is computed: exact or a summary +//! ([`HowToComputeAggregateMeasure`]). Each target has exactly one exact option: +//! its exact accumulator states (`ExactAccumulator`; `[Sum, Count]` for +//! `avg`) when they can be built, otherwise plain `Exact`. Whether they are +//! materialized is decided in Stage 2. +//! - A computation that fuses a target with the targets beneath it is the +//! outer target's [`FusedComputation`], which absorbs the ones beneath and +//! has its own candidates: the whole-expression top-k over a per-item +//! `sum` or `count` (`Summary` of a heap family) and the nested sum, a +//! grouped `sum` over a `sum` (`ExactAccumulator([Sum])`). +//! - Summary choices here are unsized families. Adding them to memo groups +//! creates one build candidate per listed parameter configuration +//! (`parameter_list`); Pass 2 gives readouts their required accuracy. +//! - Candidates are pruned only when provably unable to run, or when the +//! exact candidate provably dominates them (a PromQL top-k's heap over its +//! instant vector, one row per item); accuracy prunes nothing here (Pass +//! 2's dead-candidate elimination does). +//! - A measure filter or `HAVING` prunes nothing. The target records the +//! measure filter (applied as `SummaryAgg.filter`) and one [`HavingEntry`] +//! per consumer `Filter` directly above the aggregate: a key-only predicate +//! applied by every consumer of the aggregate's group, across queries, may go +//! below the aggregate, anything else stays above the output (approximate +//! outputs then get guarantee `None`). So every candidate of a group +//! computes the group's own rows. +//! +//! The input must be a logical DAG: ASAP operators are rejected. A logical +//! DAG has no physical properties (#1200). No cost or accuracy model is +//! used; ranking and discarding valid candidates belong to Stage 3. +use std::collections::{HashMap, HashSet}; +use std::rc::Rc; + +use asap_types::ir::operator::arguments::{Reduction, Source}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef}; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ + NonASAPOp, OperatorNode, OperatorResultKind, Predicate, QueryRoot, SchemaDerivationError, +}; +use asap_types::summary_type::{ + ExactKind, SketchAlgorithm, StatisticKind, SummaryInputExpr, SummaryUpdate, +}; +use thiserror::Error; + +use asap_accuracy::sizing::sketch_statistic; + +use crate::pass1::realization::summary_candidates; +use asap_types::deployment_capabilities::{summary_supported, SupportedSummary}; +use asap_types::stages::{ + ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, + HowToComputeAggregateMeasure, Memo, MemoGroupId, +}; + +/// Unranked candidates of one measure and the candidates pruned for it. +#[derive(Debug, Clone, PartialEq)] +pub(super) struct ComputationChoices { + pub candidates: Vec, + pub pruned: Vec, +} + +impl ComputationChoices { + /// Prune matching candidates; a pruned exact accumulator form is replaced by + /// plain `Exact`, so the target keeps one exact option. + fn prune(&mut self, applies: impl Fn(&HowToComputeAggregateMeasure) -> bool) { + let pruned = prune_own(&mut self.candidates, applies); + self.pruned.extend(pruned); + } +} + +/// Removes from a measure's own `candidates` the ones `applies` matches +/// (never `Exact`) and returns them; a pruned exact accumulator form is +/// replaced by plain `Exact`, so the measure keeps one exact option. +fn prune_own( + candidates: &mut Vec, + applies: impl Fn(&HowToComputeAggregateMeasure) -> bool, +) -> Vec { + let (pruned, kept): (Vec<_>, _) = std::mem::take(candidates) + .into_iter() + .partition(|choice| *choice != HowToComputeAggregateMeasure::Exact && applies(choice)); + *candidates = kept; + if pruned + .iter() + .any(|choice| matches!(choice, HowToComputeAggregateMeasure::ExactAccumulator(_))) + { + candidates.insert(0, HowToComputeAggregateMeasure::Exact); + } + pruned +} + +/// Compact Pass 1 inventory over the memo groups it walked: one target per +/// measure of each aggregate group, however many queries read the group. +/// Pass 1's working form, not its output: +/// [`pass1_memo`](crate::pass1::pass1_memo) +/// builds, prunes and attaches it to the memo groups. Public only to tests +/// (feature `test-support`). +#[derive(Debug, Clone)] +pub struct PerComputationCandidates { + /// Each query's root over the memo groups' logical nodes, in workload + /// order: a group several queries read is one node. Whole-workload + /// candidates are these roots with chosen candidates substituted + /// (test support only). + #[cfg(any(test, feature = "test-support"))] + pub roots: Vec, + /// Each memo group's logical node, by group index. + pub(crate) logical_nodes: Rc<[Rc]>, + /// The targets; a whole-workload choice has one digit per target, + /// ranging over [`ComputationTarget::choice_count`]. + pub targets: Vec, +} + +impl PerComputationCandidates { + /// The `Aggregate` node of `target`'s memo group. + #[cfg(any(test, feature = "test-support"))] + pub fn aggregate(&self, target: &ComputationTarget) -> &Rc { + &self.logical_nodes[target.aggregate_group.0 as usize] + } + + /// The number of whole-workload candidates: the size of the Cartesian + /// product of the targets' candidates, where a target with a fused + /// computation and the targets beneath that it absorbs count as their + /// own combinations plus the fused computation's candidates (the + /// absorbed targets are not computed then), saturating at `usize::MAX`. + /// A target with `n` candidates and `f` fused ones counts `n · Π m + f`, + /// the product over the targets it absorbs that no other of them + /// absorbs, each counted the same way. It is exactly the number of + /// choices enumeration keeps ([`Self::absorbed_choice_varies`]). + #[cfg(any(test, feature = "test-support"))] + pub fn combinations(&self) -> usize { + let absorbed: HashSet = absorbed_targets(&self.targets).collect(); + (0..self.targets.len()) + .filter(|index| !absorbed.contains(index)) + .map(|index| self.combinations_of(index)) + .fold(1usize, usize::saturating_mul) + } + + /// [`Self::combinations`] of target `index` with the targets it absorbs. + #[cfg(any(test, feature = "test-support"))] + fn combinations_of(&self, index: usize) -> usize { + let target = &self.targets[index]; + let n = target.ways_to_compute.len(); + let Some(fused) = &target.fused_computation else { + return n; + }; + let absorbed_by_another = |a: usize| { + fused.absorbed_targets.iter().any(|&b| { + self.targets[b] + .fused_computation + .as_ref() + .is_some_and(|inner| inner.absorbed_targets.contains(&a)) + }) + }; + fused + .absorbed_targets + .iter() + .filter(|&&a| !absorbed_by_another(a)) + .map(|&a| self.combinations_of(a)) + .fold(n, usize::saturating_mul) + .saturating_add(fused.ways_to_compute.len()) + } + + /// Whether `choice` picks a fused computation and a candidate other + /// than the first for a target it absorbs: the same whole-workload + /// candidate as the one with the first, so enumeration skips it. + #[cfg(any(test, feature = "test-support"))] + pub fn absorbed_choice_varies(&self, choice: &[usize]) -> bool { + self.targets.iter().zip(choice).any(|(target, &index)| { + target + .computation_for(index) + .and_then(|(_, absorbed)| absorbed) + .is_some_and(|inner| inner.iter().any(|&i| choice[i] != 0)) + }) + } + + /// This inventory without the summary candidates the executor does not + /// support (`supported_summaries`, the deployment's + /// `DeploymentCapabilities::supported_summaries`; `None` supports + /// every one): a target's own are removed, and so are a fused + /// computation's (with all of them removed, the fusion is not offered). + /// The readout is the statistic the measure's estimate reads (the outer measure's for a + /// fused computation). With every summary supported it is unchanged. + pub fn without_unsupported_summaries( + mut self, + supported_summaries: Option<&[SupportedSummary]>, + ) -> Self { + for target in self.targets.iter_mut() { + let aggregate = &self.logical_nodes[target.aggregate_group.0 as usize]; + let Some(NonASAPOp::Aggregate { measures, .. }) = aggregate.non_asap() else { + continue; + }; + let Some(readout) = sketch_statistic(&measures[target.measure_index]).map(|s| s.kind()) + else { + continue; + }; + let unsupported = |candidate: &HowToComputeAggregateMeasure| { + matches!(candidate, HowToComputeAggregateMeasure::Summary(family) + if !summary_supported(supported_summaries, family, readout)) + }; + prune_own(&mut target.ways_to_compute, unsupported); + if let Some(fused) = &mut target.fused_computation { + fused.ways_to_compute.retain(|way| !unsupported(way)); + } + } + self + } +} + +/// The indices of `targets` some target's fused computation absorbs. +#[cfg(any(test, feature = "test-support"))] +fn absorbed_targets(targets: &[ComputationTarget]) -> impl Iterator + '_ { + targets + .iter() + .filter_map(|target| target.fused_computation.as_ref()) + .flat_map(|fused| fused.absorbed_targets.iter().copied()) +} + +#[derive(Debug, Error)] +pub enum PerComputationCandidateError { + #[error(transparent)] + Structure(#[from] SchemaDerivationError), + #[error("logical candidate input already contains ASAP operators")] + ContainsAsap, + #[error("choice must name one listed candidate per target")] + InvalidChoice, + #[error("unsupported substitution in logical ASAP-aware optimization: {0}")] + Unsupported(&'static str), +} + +/// Enumerate exact, rewrite and summary choices of one measure in stable catalog +/// order, without ranking or empirical assessment. A summary is not a claim that +/// a deployment meets the measure's accuracy requirement. The one exact option +/// comes first: the exact accumulator form if the intent has one, else `Exact`. +pub(super) fn computation_candidates_for_intent(intent: &AggIntent) -> ComputationChoices { + let states = match intent { + AggIntent::Count => vec![ExactKind::Count], + AggIntent::Sum { .. } => vec![ExactKind::Sum], + AggIntent::Min { .. } => vec![ExactKind::Min], + AggIntent::Max { .. } => vec![ExactKind::Max], + AggIntent::Rate => vec![ExactKind::Rate], + AggIntent::IRate => vec![ExactKind::IRate], + AggIntent::Increase => vec![ExactKind::Increase], + AggIntent::Avg { .. } => vec![ExactKind::Sum, ExactKind::Count], + _ => vec![], + }; + let exact = if states.is_empty() { + HowToComputeAggregateMeasure::Exact + } else { + HowToComputeAggregateMeasure::ExactAccumulator(states) + }; + let mut choices = ComputationChoices { + candidates: vec![exact], + pruned: vec![], + }; + if sketch_statistic(intent).is_some() { + let summaries = summary_candidates(intent).into_iter(); + if matches!(intent, AggIntent::Quantile { q, .. } if !(0.0..=1.0).contains(q)) { + choices + .pruned + .extend(summaries.map(HowToComputeAggregateMeasure::Summary)); + } else { + choices + .candidates + .extend(summaries.map(HowToComputeAggregateMeasure::Summary)); + } + } + choices +} + +/// Whether an `avg` measure's input may be null, so `count` rows differ from +/// counted values. An unresolvable input is treated as possibly null. +fn nullable_avg_input(node: &OperatorNode, intent: &AggIntent) -> bool { + let Some(NonASAPOp::Aggregate { child, .. }) = node.non_asap() else { + return true; + }; + let column = match intent.input_cols().first() { + Some(column) => Some(*column), + None => resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok(), + }; + column + .and_then(|column| child.schema.fields.get(column)) + .is_none_or(|field| field.nullable) +} + +/// Whether `having` over `aggregate`'s output reads only `by(...)` group keys. +/// A global aggregate (no group keys) never qualifies: its one output row +/// exists even when no input row does, so no row predicate can stand in. +fn key_only_having(aggregate: &OperatorNode, having: &Predicate) -> bool { + let keys = match aggregate.non_asap() { + Some(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(by), + .. + }) if !by.is_without() => by.len(), + _ => 0, + }; + keys > 0 + && having.0.operator_refs().is_empty() + && having.0.columns_referenced().iter().all(|id| *id < keys) +} + +fn placement(below: bool) -> HavingPlacement { + if below { + HavingPlacement::BelowAggregate + } else { + HavingPlacement::AboveOutput + } +} + +/// Each memo group's expression as an operator-node tree, by group id: the +/// trees target discovery still walks. `Memo::logical_node_trees` is +/// deleted; discovery reads the groups in #1207 PR 3. +fn logical_node_trees(memo: &Memo) -> Vec> { + let _ = memo; + todo!("#1207 PR 3: discover targets over the memo groups, not node trees") +} + +/// Discover one target per measure of each eligible aggregate memo group of +/// `dag`, including operator plans read by scalar roots: Pass 1 walks the +/// groups, so an aggregate several queries read is one target per measure. +/// An aggregate measure is eligible when it has a realization besides exact +/// execution, even if every such realization is pruned; the prune reasons +/// are reported on the target. Measure filters and `HAVING` keep every +/// candidate; the target records how they are applied. A group's consumers +/// are its parent groups and the query roots that read it; a consumer +/// `Filter`'s key-only `HAVING` is pushed below only when every consumer of +/// the group applies the same predicate, so every candidate of a shared +/// aggregate computes the group's own rows (#1089). +pub fn generate_per_computation_candidates( + memo: &Memo, +) -> Result { + let logical_nodes: Rc<[Rc]> = logical_node_trees(memo).into(); + let group_of: HashMap<*const OperatorNode, MemoGroupId> = logical_nodes + .iter() + .enumerate() + .map(|(g, node)| (Rc::as_ptr(node), MemoGroupId(g as u32))) + .collect(); + // Each query's root over the groups' logical nodes, in workload order: + // a group several queries read is one node. + let node = |g: &MemoGroupId| Rc::clone(&logical_nodes[g.0 as usize]); + let roots: Vec = memo + .queries + .iter() + .map(|query| match &query.root { + QueryRoot::Operator(g) => QueryRoot::Operator(node(g)), + QueryRoot::Scalar(expr) => QueryRoot::Scalar(expr.map_operator_refs(&mut |g| node(g))), + }) + .collect(); + let mut seen = HashSet::new(); + // Each aggregate group. + let mut aggregates = Vec::new(); + // Every consumer of each group's node: `Some(parent)`, or `None` for a + // query root or scalar reference. + let mut consumers: HashMap<*const OperatorNode, Vec>>> = HashMap::new(); + for root in &roots { + root.validate_structure()?; + let operators = match root { + QueryRoot::Operator(node) => vec![node], + QueryRoot::Scalar(expr) => expr.operator_refs(), + }; + for root in operators { + if root.contains_asap() { + return Err(PerComputationCandidateError::ContainsAsap); + } + consumers.entry(Rc::as_ptr(root)).or_default().push(None); + for node in OperatorNode::reachable(root) { + if !seen.insert(Rc::as_ptr(&node)) { + continue; + } + for child in node.children() { + consumers + .entry(Rc::as_ptr(child)) + .or_default() + .push(Some(Rc::clone(&node))); + } + if matches!(node.non_asap(), Some(NonASAPOp::Aggregate { .. })) { + aggregates.push(node); + } + } + } + } + let mut targets = Vec::new(); + for node in aggregates { + let Some(NonASAPOp::Aggregate { + measures, + filters, + having, + .. + }) = node.non_asap() + else { + continue; + }; + let aggregate_group = group_of[&Rc::as_ptr(&node)]; + let mut entries: Vec = having + .iter() + .map(|having| HavingEntry { + filter: None, + predicate: having.clone(), + placement: placement(key_only_having(&node, having)), + }) + .collect(); + let parents = &consumers[&Rc::as_ptr(&node)]; + // Only a `Filter` whose input is this aggregate is its `HAVING`; one + // that reads it from a scalar expression stays a plain consumer. + let mut filters_above: Vec<(&Rc, &Predicate)> = Vec::new(); + for parent in parents.iter().flatten() { + if let Some(NonASAPOp::Filter { child, pred }) = parent.non_asap() { + if Rc::ptr_eq(child, &node) + && !filters_above + .iter() + .any(|(filter, _)| Rc::ptr_eq(filter, parent)) + { + filters_above.push((parent, pred)); + } + } + } + for (filter, predicate) in &filters_above { + // Every consumer of the group, across queries, applies it. + let shared_by_all = filters_above.len() == parents.len() + && filters_above.iter().all(|(_, other)| other == predicate); + entries.push(HavingEntry { + filter: Some(Rc::clone(filter)), + predicate: (*predicate).clone(), + placement: placement(shared_by_all && key_only_having(&node, predicate)), + }); + } + for (measure, intent) in measures.iter().enumerate() { + let mut choices = computation_candidates_for_intent(intent); + if choices.candidates == [HowToComputeAggregateMeasure::Exact] + && choices.pruned.is_empty() + { + continue; + } + if matches!(intent, AggIntent::Avg { .. }) && nullable_avg_input(&node, intent) { + choices.prune(|choice| { + matches!(choice, HowToComputeAggregateMeasure::ExactAccumulator(_)) + }); + } + if matches!(intent, AggIntent::Count) + && node.children()[0].result_kind == OperatorResultKind::Relation + { + choices.prune(|choice| matches!(choice, HowToComputeAggregateMeasure::Summary(_))); + } + targets.push(ComputationTarget { + aggregate_group, + measure_index: measure, + ways_to_compute: choices.candidates, + measure_filter: filters.get(measure).cloned().flatten(), + having_filters: entries.clone(), + fused_computation: None, + }); + } + } + add_whole_expression_targets(&mut targets, &logical_nodes); + prune_topk_heaps_over_one_row_per_item(&mut targets, &logical_nodes); + add_nested_sum_targets(&mut targets, &logical_nodes); + close_absorptions(&mut targets); + Ok(PerComputationCandidates { + #[cfg(any(test, feature = "test-support"))] + roots, + logical_nodes, + targets, + }) +} + +/// Adds to each fused computation's `absorbed_targets` the targets its absorbed +/// targets' own fused computations absorb, transitively: none of them is +/// computed when the fused computation is picked, so enumeration keeps only +/// their first choice ([`PerComputationCandidates::absorbed_choice_varies`]) +/// and [`PerComputationCandidates::combinations`] counts exactly those. +fn close_absorptions(targets: &mut [ComputationTarget]) { + loop { + let mut changed = false; + for t in 0..targets.len() { + let Some(fused) = &targets[t].fused_computation else { + continue; + }; + let beneath: Vec = fused + .absorbed_targets + .iter() + .filter_map(|&a| targets[a].fused_computation.as_ref()) + .flat_map(|inner| inner.absorbed_targets.iter().copied()) + .filter(|b| !fused.absorbed_targets.contains(b)) + .collect(); + if let Some(fused) = targets[t] + .fused_computation + .as_mut() + .filter(|_| !beneath.is_empty()) + { + for b in beneath { + if !fused.absorbed_targets.contains(&b) { + fused.absorbed_targets.push(b); + } + } + changed = true; + } + } + if !changed { + return; + } + } +} + +/// Nested-sum merge: a grouped `sum` over the value of a `sum` target +/// beneath it gets a fused computation ([`ComputationTarget::fused_computation`]) +/// with the one candidate `ExactAccumulator([Sum])`: one exact `Sum` +/// accumulator over the inner sum's input, grouped by the outer keys (the +/// sum of the inner groups' sums is the sum of their rows), absorbing the +/// inner target. Offered whatever else reads the inner sum: it is one more +/// candidate of the outer sum's group, and the inner sum's group keeps its +/// own candidates for its other consumers. Applied when neither has a filter +/// or `HAVING` +/// ([`nested_sum_input`]), and when the outer sum is not itself absorbed by +/// a merge and has no fused computation yet, and the inner sum has none, so +/// merges do not stack. An outer sum a whole-expression top-k absorbs +/// (`topk(k, sum by (G) (sum_over_time(m[W])))`) keeps its merge for the +/// top-k's own candidates. +fn add_nested_sum_targets(targets: &mut [ComputationTarget], logical_nodes: &[Rc]) { + let aggregate = |target: &ComputationTarget| &logical_nodes[target.aggregate_group.0 as usize]; + // The targets a merge absorbs. + let mut absorbed: HashSet = HashSet::new(); + for t in 0..targets.len() { + let target = &targets[t]; + if absorbed.contains(&t) + || target.fused_computation.is_some() + || !target.having_filters.is_empty() + { + continue; + } + let Some(NonASAPOp::Aggregate { child, .. }) = aggregate(target).non_asap() else { + continue; + }; + let Some(inner) = targets + .iter() + .position(|other| Rc::ptr_eq(aggregate(other), child)) + else { + continue; + }; + // Merges do not stack: the inner sum must not have a fused + // computation of its own. + if !targets[inner].having_filters.is_empty() + || targets[inner].fused_computation.is_some() + || nested_sum_input(aggregate(target)).is_none() + { + continue; + } + absorbed.insert(inner); + let merge = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); + targets[t].fused_computation = Some(FusedComputation { + absorbed_targets: vec![inner], + ways_to_compute: vec![merge], + }); + } +} + +/// The input rows, the inner sum and the outer grouping (moved onto the +/// input rows by column, [`Reduction::partitions_over`]) of a nested-sum merge of +/// `target`, or `None` when it does not apply: `target` is a grouped `sum` +/// (not by `without`) of the value column of a per-series or grouped `sum` +/// directly beneath it, neither filtered nor with `HAVING`, and not grouped +/// by the metric name a per-series inner sum drops. +pub(super) fn nested_sum_input( + target: &OperatorNode, +) -> Option<(Rc, AggIntent, Reduction)> { + use asap_types::ir::operator::non_asap::any_measure_filtered; + let Some(NonASAPOp::Aggregate { + child: inner, + reduction: reduction @ Reduction::Reduce(by), + measures, + filters, + having: None, + .. + }) = target.non_asap() + else { + return None; + }; + let [AggIntent::Sum { col }] = measures.as_slice() else { + return None; + }; + let Some(NonASAPOp::Aggregate { + child: rows, + reduction: inner_reduction, + measures: inner_measures, + filters: inner_filters, + having: None, + .. + }) = inner.non_asap() + else { + return None; + }; + let [inner_sum @ AggIntent::Sum { .. }] = inner_measures.as_slice() else { + return None; + }; + if by.is_without() || any_measure_filtered(filters) || any_measure_filtered(inner_filters) { + return None; + } + // The inner sum's value column: a per-series sum keeps its labels and + // carries the value as the sample value; a grouped one ends with it. + let value = match inner_reduction { + Reduction::PerEntity => resolve_column_ref(&ColumnRef::SampleValue, &inner.schema).ok()?, + Reduction::Reduce(keys) if !keys.is_without() => keys.len(), + Reduction::Reduce(_) => return None, + }; + let summed = match col { + None => resolve_column_ref(&ColumnRef::SampleValue, &inner.schema).ok()?, + Some(column) => *column, + }; + if summed != value || by.keys().contains(&value) { + return None; + } + // A per-series sum drops the metric name, which the raw rows keep: + // grouping them by it would name groups the nested sums leave unnamed. + if matches!(inner_reduction, Reduction::PerEntity) + && by.keys().iter().any(|&key| { + inner + .schema + .fields + .get(key) + .is_some_and(|f| f.name == "__name__") + }) + { + return None; + } + let partitions = reduction.partitions_over(&inner.schema, &rows.schema)?; + Some((Rc::clone(rows), inner_sum.clone(), partitions)) +} + +/// Whole-expression top-k (Example 1's Q2): a top-k over a per-item `sum` +/// or `count` that is itself a target gets a fused computation +/// ([`ComputationTarget::fused_computation`]) absorbing the inner target, whose +/// candidates are the top-k's heap summaries (Count-Min and CountSketch +/// with a heap), each one heap over the inner aggregate's input, keyed by +/// the ranked item and weighted by the summed value. Those heaps over the +/// exact per-item result leave the top-k's own candidates: the fused +/// computation replaces them. Over a grouped sum of per-series sums or +/// counts of raw data samples, the heap reads those samples and absorbs the +/// per-series target too. Offered whatever else reads the inner aggregate: +/// it is one more candidate of the top-k's group, and the absorbed groups +/// keep their own candidates for their other consumers. Applied when neither +/// has a filter or `HAVING`, and when the keyed-additive rule realizes it +/// ([`whole_expression_input`]). +fn add_whole_expression_targets( + targets: &mut [ComputationTarget], + logical_nodes: &[Rc], +) { + let aggregate = |target: &ComputationTarget| &logical_nodes[target.aggregate_group.0 as usize]; + for t in 0..targets.len() { + let target = &targets[t]; + let Some(NonASAPOp::Aggregate { child, .. }) = aggregate(target).non_asap() else { + continue; + }; + let Some(inner) = targets + .iter() + .position(|other| Rc::ptr_eq(aggregate(other), child)) + else { + continue; + }; + if target.measure_filter.is_some() + || !target.having_filters.is_empty() + || targets[inner].measure_filter.is_some() + || !targets[inner].having_filters.is_empty() + { + continue; + } + let Some((input, ..)) = whole_expression_input(aggregate(target)) else { + continue; + }; + // A heap over the raw data samples beneath a grouped sum of per-series + // sums or counts also absorbs the per-series target. + let mut absorbs = vec![inner]; + if let Some(NonASAPOp::Aggregate { + child: per_series, .. + }) = aggregate(&targets[inner]).non_asap() + { + let reads_below = per_series + .children() + .iter() + .any(|&rows| Rc::ptr_eq(rows, &input)); + if reads_below { + absorbs.extend( + targets + .iter() + .position(|other| Rc::ptr_eq(aggregate(other), per_series)), + ); + } + } + let (heaps, own): (Vec<_>, Vec<_>) = + targets[t] + .ways_to_compute + .iter() + .cloned() + .partition(|candidate| { + matches!(candidate, HowToComputeAggregateMeasure::Summary(family) + if family.capability().answers(StatisticKind::TopK)) + }); + if heaps.is_empty() { + continue; + } + let target = &mut targets[t]; + target.ways_to_compute = own; + target.fused_computation = Some(FusedComputation { + absorbed_targets: absorbs, + ways_to_compute: heaps, + }); + } +} + +/// Prunes the own heap candidates of each PromQL top-k: the heap reads +/// one row per item, so exact is no costlier; a fused whole-expression +/// heap ([`add_whole_expression_targets`]) has already moved out of them. +fn prune_topk_heaps_over_one_row_per_item( + targets: &mut [ComputationTarget], + logical_nodes: &[Rc], +) { + for target in targets.iter_mut() { + let Some(NonASAPOp::Aggregate { + measures, child, .. + }) = logical_nodes[target.aggregate_group.0 as usize].non_asap() + else { + continue; + }; + if !matches!( + measures.get(target.measure_index), + Some(AggIntent::TopK { .. }) + ) { + continue; + } + // A PromQL top-k ranks an instant vector: one row per series or + // group. (A SQL top-k keeps its heaps.) + if !reads_only_time_series(child) { + continue; + } + prune_own(&mut target.ways_to_compute, |candidate| { + matches!(candidate, HowToComputeAggregateMeasure::Summary(family) + if family.capability().answers(StatisticKind::TopK)) + }); + } +} + +/// Whether every scan below `node` reads a time series (PromQL rows). +fn reads_only_time_series(node: &OperatorNode) -> bool { + match node.non_asap() { + Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), + _ => node.children().iter().all(|c| reads_only_time_series(c)), + } +} + +/// The input, update and partitions of a whole-expression top-k over +/// `target`'s inner aggregate, by the keyed-additive rule, or `None` when it +/// does not apply. Rows that carry the full series identity rank it as a +/// column, as [`summary_update`] does. The partitions are `target`'s, moved +/// onto the input the heap reads ([`Reduction::partitions_over`]). +pub(super) fn whole_expression_input( + target: &OperatorNode, +) -> Option<(Rc, SummaryUpdate, Reduction)> { + use crate::pass1::realization::realize_keyed_additive_summary_input; + let Some(NonASAPOp::Aggregate { + child, + reduction, + measures, + filters, + having: None, + .. + }) = target.non_asap() + else { + return None; + }; + let [intent @ AggIntent::TopK { .. }] = measures.as_slice() else { + return None; + }; + if asap_types::ir::operator::non_asap::any_measure_filtered(filters) { + return None; + } + // The rule reads only the family's algorithm, and rejects Count-Min over + // weights it cannot prove non-negative. Ask with CountSketch so one + // update serves both heap families; plan selection decides whether + // Count-Min is admissible, as for every other Count-Min candidate. + let family = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap); + let realized = realize_keyed_additive_summary_input(intent, &family, reduction, child)?; + let mut input = realized.input; + let per_series = matches!( + child.non_asap(), + Some(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + .. + }) + ); + let partitions = reduction.partitions_over(&child.schema, &realized.child.schema)?; + if per_series && realized.child.schema.has_promql_series_identity() { + input.item = Some(asap_types::ir::schema::topk::series_identity_item( + &realized.child.schema, + partitions.group_keys().map_or(&[][..], |keys| keys.keys()), + )); + } else if realized.child.schema.closed + && matches!(input.item, Some(SummaryInputExpr::EntityIdentity(_))) + { + // An encoded label set needs an open PromQL schema. PromQL rows that + // carry the series identity are closed, but their inner group labels + // are item columns. SQL group keys are explicit columns too, so their + // keyed heap can consume raw rows without a per-key aggregate first. + return None; + } + Some((realized.child, input, partitions)) +} + +#[cfg(test)] +mod tests { + + use super::*; + use asap_types::accuracy::{AccuracyGuarantee, AccuracyTarget}; + use asap_types::ir::operator::arguments::{BinaryOpKind, JoinKind}; + use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; + use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; + use asap_types::ir::{BinaryOperator, ExprSemantics, Operator, ProjectItem, ScalarExpr}; + use asap_types::summary_type::{NonNegativeWeightProof, WeightDomain}; + + use asap_accuracy::propagation::raw_data_samples_of_metric; + + use asap_accuracy::composition::PRODUCT_OPERAND_DOMAINS; + + use asap_accuracy::test_support::{ + composed_guarantee, estimate_has_bound, scalar_guarantee, source_distinct_guarantee, + QUOTIENT_OPERAND_DOMAINS, + }; + use asap_types::accuracy::{AccuracyMetric, BoundExpr, ProbabilityExpr}; + + use asap_types::summary_type::SketchStatistic; + /// Approximate count keeps one exact option, its accumulator, beside the + /// universal summary; plain exact execution is not offered. + #[test] + fn approximate_count_keeps_one_exact_accumulator_and_universal_choices() { + let choices = computation_candidates_for_intent(&AggIntent::Count).candidates; + assert!(!choices.contains(&HowToComputeAggregateMeasure::Exact)); + assert_eq!( + choices[0], + HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count]) + ); + assert!(choices.contains(&HowToComputeAggregateMeasure::Summary( + SketchAlgorithm::UnivMon + ))); + } + + /// Count Sketch's signed counters give no total, so Pass 1 offers no + /// Count Sketch for a count, and every count candidate it offers + /// answers the total. + #[test] + fn a_count_offers_no_count_sketch() { + let choices = computation_candidates_for_intent(&AggIntent::Count).candidates; + assert!(!choices.contains(&HowToComputeAggregateMeasure::Summary( + SketchAlgorithm::CountSketch + ))); + let total = sketch_statistic(&AggIntent::Count).unwrap(); + for choice in &choices { + if let HowToComputeAggregateMeasure::Summary(algorithm) = choice { + assert!( + asap_types::summary_type::SummaryTypeCapability::of(&FieldDataType::Sketch( + *algorithm + )) + .answers(&total), + "{algorithm:?}" + ); + } + } + } + + /// Guarantee composition through scalar and vector arithmetic (#669). + mod guarantee_composition { + use super::*; + use crate::test_support::lower_promql; + use asap_types::accuracy::AccuracyTarget; + + fn with_bound(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { + AccuracyGuarantee::new( + metric, + BoundExpr::Constant { value: bound }, + ProbabilityExpr::Constant { value: delta }, + vec![], + ) + } + fn abs(bound: f64, delta: f64) -> AccuracyGuarantee { + with_bound(AccuracyMetric::ABSOLUTE, bound, delta) + } + fn rel(bound: f64) -> AccuracyGuarantee { + with_bound(AccuracyMetric::Relative, bound, 0.0) + } + /// `sum(m)` carrying `guarantee`. + fn node(guarantee: AccuracyGuarantee) -> Rc { + node_over("sum(m)", guarantee) + } + /// `query` carrying `guarantee`. + fn node_over(query: &str, guarantee: AccuracyGuarantee) -> Rc { + let node = lower_promql(query, AccuracyTarget::exact()); + Rc::new((*node).clone().with_guarantee(Some(guarantee))) + } + fn scalar(guarantee: AccuracyGuarantee) -> ScalarExpr { + ScalarExpr::PromqlScalarFromVector(node(guarantee)) + } + fn lit(value: f64) -> ScalarExpr { + ScalarExpr::literal_f64(value) + } + fn arith(op: ArithmeticOpKind, left: ScalarExpr, right: ScalarExpr) -> ScalarExpr { + ScalarExpr::Arithmetic { + op, + left: Box::new(left), + right: Box::new(right), + semantics: ExprSemantics::Promql, + } + } + fn call(name: &str, args: Vec) -> ScalarExpr { + ScalarExpr::FunctionCall { + name: name.into(), + args, + } + } + fn of(expr: &ScalarExpr) -> Option { + scalar_guarantee(expr, &|_| None) + } + fn bound(g: &AccuracyGuarantee) -> f64 { + g.spec.error.evaluate().expect("a resolved bound") + } + fn delta(g: &AccuracyGuarantee) -> f64 { + g.spec + .failure_probability + .evaluate() + .expect("a resolved probability") + } + fn close(a: f64, b: f64) -> bool { + (a - b).abs() < 1e-12 + } + + /// Absolute errors add through `x ± y`, with the union bound on the + /// failure probability; `−x` and `x ± c` keep the error, `c · x` and + /// `x / c` scale it by `|c|` and `1/|c|`, division by a literal zero + /// has none. + #[test] + fn absolute_errors_compose_through_arithmetic() { + use ArithmeticOpKind::*; + let (x, y) = (scalar(abs(0.01, 0.001)), scalar(abs(0.02, 0.002))); + for op in [Add, Sub] { + let g = of(&arith(op.clone(), x.clone(), y.clone())).unwrap(); + assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE, "{op:?}"); + assert!(close(bound(&g), 0.03) && close(delta(&g), 0.003), "{op:?}"); + for shifted in [ + arith(op.clone(), x.clone(), lit(5.0)), + arith(op.clone(), lit(5.0), x.clone()), + arith(op.clone(), x.clone(), ScalarExpr::EvalTimestamp), + ] { + let g = of(&shifted).unwrap(); + assert!(close(bound(&g), 0.01) && close(delta(&g), 0.001), "{op:?}"); + } + } + let negated = ScalarExpr::Negative { + expr: Box::new(x.clone()), + semantics: ExprSemantics::Promql, + }; + assert!(close(bound(&of(&negated).unwrap()), 0.01)); + for (expr, expected) in [ + (arith(Mul, lit(-3.0), x.clone()), 0.03), + (arith(Mul, x.clone(), lit(3.0)), 0.03), + (arith(Div, x.clone(), lit(-4.0)), 0.0025), + ] { + let g = of(&expr).unwrap(); + assert!( + close(bound(&g), expected) && close(delta(&g), 0.001), + "{expr:?}" + ); + } + for none in [ + arith(Div, x.clone(), lit(0.0)), + arith(Mul, x.clone(), ScalarExpr::EvalTimestamp), + arith(Pow, x.clone(), lit(2.0)), + ] { + assert_eq!(of(&none), None, "{none:?}"); + } + } + + /// A product of two estimates has an absolute bound + /// `|x|·e_y + |y|·e_x + e_x·e_y`, and a quotient with an approximate + /// denominator one that needs the denominator bounded away from + /// zero: both need operand domains, so Stage 1 leaves them symbolic + /// for plan selection to bind. Relative errors need no domain: a + /// product has the closed form `(1 + a)(1 + b) − 1`, a quotient + /// `(a + b) / (1 − b)` (`a = 0` for an exact numerator). The failure + /// probability is the union bound. A rank error composes to none. + #[test] + fn products_and_quotients_of_estimates_have_closed_or_symbolic_bounds() { + use ArithmeticOpKind::*; + let unknown = |statistic: &str| BoundExpr::Unknown { + statistic: statistic.into(), + }; + let (x, y) = (scalar(abs(0.01, 0.001)), scalar(abs(0.02, 0.002))); + let product = of(&arith(Mul, x.clone(), y.clone())).unwrap(); + assert_eq!(product.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); + assert_eq!(product.spec.error, unknown(PRODUCT_OPERAND_DOMAINS)); + assert!(close(delta(&product), 0.003)); + for quotient in [ + arith(Div, x.clone(), y.clone()), + arith(Div, lit(1.0), x.clone()), + arith(Div, x.clone(), scalar(rel(0.01))), + ] { + let g = of("ient).unwrap(); + assert_eq!( + g.spec.accuracy_metric, + AccuracyMetric::ABSOLUTE, + "{quotient:?}" + ); + assert_eq!( + g.spec.error, + unknown(QUOTIENT_OPERAND_DOMAINS), + "{quotient:?}" + ); + } + let ratio = of(&arith( + Div, + scalar(with_bound(AccuracyMetric::Relative, 0.01, 0.001)), + scalar(with_bound(AccuracyMetric::Relative, 0.02, 0.002)), + )) + .unwrap(); + assert_eq!(ratio.spec.accuracy_metric, AccuracyMetric::Relative); + assert!(close(bound(&ratio), (0.01 + 0.02) / (1.0 - 0.02))); + assert!(close(delta(&ratio), 0.003)); + let reciprocal = of(&arith(Div, lit(2.0), scalar(rel(0.02)))).unwrap(); + assert_eq!(reciprocal.spec.accuracy_metric, AccuracyMetric::Relative); + assert!(close(bound(&reciprocal), 0.02 / (1.0 - 0.02))); + // A denominator bound of 1 or more leaves the ratio unbounded. + assert_eq!(of(&arith(Div, scalar(rel(0.01)), scalar(rel(1.0)))), None); + // A relative bound still symbolic in Stage 1 has no ratio rule. + let symbolic = AccuracyGuarantee::new( + AccuracyMetric::Relative, + unknown("values_sign"), + ProbabilityExpr::Zero, + vec![], + ); + assert_eq!(of(&arith(Div, scalar(symbolic), scalar(rel(0.01)))), None); + let relative = of(&arith(Mul, scalar(rel(0.01)), scalar(rel(0.02)))).unwrap(); + assert_eq!(relative.spec.accuracy_metric, AccuracyMetric::Relative); + assert!(close(bound(&relative), 0.01 + 0.02 + 0.0002)); + let rank = scalar(with_bound(AccuracyMetric::Rank, 0.01, 0.001)); + assert_eq!(of(&arith(Mul, rank.clone(), x.clone())), None); + assert_eq!(of(&arith(Div, x, rank)), None); + } + + /// A relative error survives negation and scaling by an exact value, + /// but not a shift (`x + c`) or a sum or difference of estimates + /// (cancellation); a rank error composes to no guarantee (a rank + /// interval counts as none in the MVP). + #[test] + fn relative_and_rank_errors_compose_only_where_sound() { + use ArithmeticOpKind::*; + let x = scalar(rel(0.01)); + for kept in [ + ScalarExpr::Negative { + expr: Box::new(x.clone()), + semantics: ExprSemantics::Promql, + }, + arith(Mul, x.clone(), lit(-2.0)), + arith(Div, x.clone(), ScalarExpr::EvalTimestamp), + ] { + let g = of(&kept).unwrap(); + assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative, "{kept:?}"); + assert!(close(bound(&g), 0.01), "{kept:?}"); + } + for none in [ + arith(Add, x.clone(), lit(1.0)), + arith(Sub, x.clone(), scalar(rel(0.01))), + ] { + assert_eq!(of(&none), None, "{none:?}"); + } + let rank = scalar(with_bound(AccuracyMetric::Rank, 0.01, 0.001)); + for none in [ + arith(Sub, rank.clone(), rank.clone()), + arith(Mul, rank.clone(), lit(2.0)), + call("promql_clamp_max", vec![rank, lit(1.0)]), + ] { + assert_eq!(of(&none), None, "{none:?}"); + } + } + + /// `clamp`, `clamp_min`, `clamp_max` and `abs` are 1-Lipschitz and + /// monotone in each argument: the result keeps the largest absolute + /// (or relative) error of its arguments, with the union bound on the + /// failure probability. + #[test] + fn clamp_and_abs_keep_the_largest_argument_error() { + let x = scalar(abs(0.01, 0.001)); + for expr in [ + call("promql_clamp", vec![x.clone(), lit(0.0), lit(10.0)]), + call("promql_clamp_min", vec![x.clone(), lit(0.0)]), + call("promql_clamp_max", vec![x.clone(), lit(10.0)]), + call("promql_abs", vec![x.clone()]), + call("abs", vec![x.clone()]), + ] { + let g = of(&expr).unwrap(); + assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE, "{expr:?}"); + assert!( + close(bound(&g), 0.01) && close(delta(&g), 0.001), + "{expr:?}" + ); + } + let bounded = of(&call( + "promql_clamp", + vec![x.clone(), scalar(abs(0.03, 0.002)), lit(10.0)], + )) + .unwrap(); + assert!(close(bound(&bounded), 0.03) && close(delta(&bounded), 0.003)); + let relative = of(&call( + "promql_clamp", + vec![scalar(rel(0.01)), scalar(rel(0.02)), lit(10.0)], + )) + .unwrap(); + assert_eq!(relative.spec.accuracy_metric, AccuracyMetric::Relative); + assert!(close(bound(&relative), 0.02)); + let mixed = call("promql_clamp_min", vec![x, scalar(rel(0.02))]); + assert_eq!(of(&mixed), None); + assert_eq!(of(&call("promql_floor", vec![scalar(rel(0.01))])), None); + } + + /// A comparison over an estimate has no guarantee on its output (a + /// band guarantee is later work); over exact operands it is exact. + #[test] + fn comparisons_over_estimates_have_no_guarantee() { + let compare = |left: ScalarExpr| ScalarExpr::Compare { + left: Box::new(left), + op: CompareOpKind::Gt, + right: Box::new(lit(5.0)), + semantics: ExprSemantics::Promql, + }; + assert_eq!(of(&compare(scalar(abs(0.01, 0.001)))), None); + assert!(of(&compare(scalar(AccuracyGuarantee::exact("exact")))) + .is_some_and(|g| g.is_exact())); + } + + /// A SQL scalar subquery inherits its relation's guarantee, as + /// PromQL `scalar(v)` does. + #[test] + fn a_scalar_subquery_inherits_its_guarantee() { + let g = abs(0.01, 0.001); + let expr = arith( + ArithmeticOpKind::Add, + ScalarExpr::ScalarSubquery(node(g.clone())), + lit(1.0), + ); + assert!(close(bound(&of(&expr).unwrap()), 0.01)); + } + + /// Two estimates count their failure probabilities once each (union + /// bound), even when they read one shared summary: the built-in + /// guarantees hold per estimate, not simultaneously. The same + /// planned estimate read twice fails or holds as one event, so its + /// failure probability counts once. + #[test] + fn failure_probability_counts_once_per_distinct_estimate() { + let shared = node(abs(0.01, 0.001)); + // Equal to `shared`, but planned separately. + let other = node(abs(0.01, 0.001)); + let same = ScalarExpr::PromqlScalarFromVector(shared.clone()); + let twice = of(&arith(ArithmeticOpKind::Add, same.clone(), same.clone())).unwrap(); + assert!(close(bound(&twice), 0.02) && close(delta(&twice), 0.001)); + let distinct = of(&arith( + ArithmeticOpKind::Add, + same, + ScalarExpr::PromqlScalarFromVector(other.clone()), + )) + .unwrap(); + assert!(close(bound(&distinct), 0.02) && close(delta(&distinct), 0.002)); + let vector = |lhs: &Rc, rhs: &Rc| { + let node = OperatorNode::new(Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Sub), + vector_match: None, + }, + return_bool: false, + lhs: lhs.clone(), + rhs: rhs.clone(), + })) + .unwrap(); + composed_guarantee(&node).unwrap() + }; + assert!(close(delta(&vector(&shared, &shared)), 0.001)); + assert!(close(delta(&vector(&shared, &other)), 0.002)); + } + + /// `left JOIN right ON pred`, composed. + fn join( + kind: JoinKind, + left: &Rc, + right: &Rc, + pred: ScalarExpr, + ) -> OperatorNode { + let mut node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Join { + kind, + pred: Predicate(pred), + left: left.clone(), + right: right.clone(), + })) + .unwrap(); + node.guarantee = composed_guarantee(&node); + node + } + fn column(id: usize) -> ScalarExpr { + ScalarExpr::Column(id) + } + fn equals(left: ScalarExpr, right: ScalarExpr) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(left), + op: CompareOpKind::Eq, + right: Box::new(right), + semantics: ExprSemantics::Sql, + } + } + /// `CASE WHEN condition THEN NULL ELSE otherwise END` over `child`, + /// composed. + fn case_over( + child: &OperatorNode, + condition: ScalarExpr, + otherwise: ScalarExpr, + ) -> Option { + let node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::Case { + operand: None, + branches: vec![(condition, ScalarExpr::Literal(ScalarValue::Null))], + else_expr: Some(Box::new(otherwise)), + }, + }], + qualifier: None, + child: Rc::new(child.clone()), + })) + .unwrap(); + composed_guarantee(&node) + } + /// An approximate node and an exact one, and the approximate one's + /// column count (its columns come first in a join). + fn approximate_and_exact() -> (Rc, Rc, usize) { + let approximate = node(rel(0.01)); + let exact = node_over("count(m)", AccuracyGuarantee::exact("test")); + let width = approximate.schema.fields.len(); + (approximate, exact, width) + } + + /// A join whose other input is exact and whose predicate reads only + /// exact columns keeps the approximate input's guarantee: a cross + /// join, or a key equality on the exact input's columns. + #[test] + fn join_beside_an_exact_input_on_exact_columns_keeps_the_guarantee() { + let (approximate, exact, width) = approximate_and_exact(); + for (kind, pred) in [ + ( + JoinKind::Cross, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ), + (JoinKind::Inner, equals(column(width), lit(0.0))), + (JoinKind::Left, equals(column(width), lit(0.0))), + ] { + assert_eq!( + join(kind.clone(), &approximate, &exact, pred).guarantee, + Some(rel(0.01)), + "{kind:?}" + ); + } + // Either side. + assert_eq!( + join( + JoinKind::Cross, + &exact, + &approximate, + ScalarExpr::Literal(ScalarValue::Boolean(true)) + ) + .guarantee, + Some(rel(0.01)) + ); + } + + /// A join keyed on an approximate input's column, a semi-join, or + /// two approximate inputs have no guarantee (column-level tracking + /// is #689). + #[test] + fn join_on_an_approximate_key_has_no_guarantee() { + let (approximate, exact, width) = approximate_and_exact(); + assert_eq!( + join( + JoinKind::Inner, + &approximate, + &exact, + equals(column(0), column(width)) + ) + .guarantee, + None + ); + assert_eq!( + join( + JoinKind::Semi, + &exact, + &approximate, + ScalarExpr::Literal(ScalarValue::Boolean(true)) + ) + .guarantee, + None + ); + assert_eq!( + join( + JoinKind::Cross, + &approximate, + &approximate, + ScalarExpr::Literal(ScalarValue::Boolean(true)) + ) + .guarantee, + None + ); + } + + /// A `CASE` whose condition reads only exact columns keeps each + /// branch's guarantee (a NULL branch is exact); one testing an + /// approximate column has none. + #[test] + fn case_on_exact_columns_keeps_the_branch_guarantee() { + let (approximate, exact, width) = approximate_and_exact(); + let joined = join( + JoinKind::Cross, + &approximate, + &exact, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ); + let value = approximate.schema.fields.len() - 1; + assert_eq!( + case_over(&joined, equals(column(width), lit(0.0)), column(value)), + Some(rel(0.01)) + ); + assert_eq!( + case_over(&joined, equals(column(value), lit(0.0)), column(value)), + None + ); + } + + /// `node` projecting `cols` over `child`, composed. + fn project(child: &OperatorNode, cols: Vec) -> OperatorNode { + let mut node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { + cols: cols + .into_iter() + .enumerate() + .map(|(i, expr)| ProjectItem { + alias: Some(format!("c{i}")), + expr, + }) + .collect(), + qualifier: None, + child: Rc::new(child.clone()), + })) + .unwrap(); + node.guarantee = composed_guarantee(&node); + node + } + + /// Over a join, a projection reads each column's own guarantee, + /// not the child's sample value (which may be an exact input's + /// column): `sum(m) * 1000` beside the exact side's `value` has + /// the scaled error 1000, not the approximate side's 1. A + /// projection over that computed projection has no per-column + /// guarantees, so it has none (#689). + #[test] + fn projection_over_a_join_reads_each_columns_guarantee() { + let approximate = node(abs(1.0, 0.01)); + let exact = node_over("m", AccuracyGuarantee::exact("test")); + let joined = join( + JoinKind::Cross, + &approximate, + &exact, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ); + assert_eq!(joined.guarantee, Some(abs(1.0, 0.01))); + let sum = approximate.schema.fields.len() - 1; + let value = approximate.schema.fields.len() + exact.schema.fields.len() - 1; + let scaled = project( + &joined, + vec![ + arith(ArithmeticOpKind::Mul, column(sum), lit(1000.0)), + column(value), + ], + ); + assert_eq!(scaled.guarantee.as_ref().map(bound), Some(1000.0)); + let stacked = project( + &scaled, + vec![arith(ArithmeticOpKind::Add, column(1), lit(1.0)), column(0)], + ); + assert_eq!(stacked.guarantee, None); + } + } + + fn inventory(query: &str) -> PerComputationCandidates { + let root = crate::test_support::lower_promql( + query, + AccuracyTarget::max_error(0.01).failure_probability(0.001), + ); + // Per-series state ranks each series' identity, which the rows carry + // as a column; a query over aggregated rows is planned over labels. + let root = asap_types::ir::schema::series_identity::with_promql_series_identity(&root) + .unwrap_or(root); + generate_per_computation_candidates( + &crate::stage0::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), + ) + .unwrap() + } + + fn topk_target(inventory: &PerComputationCandidates) -> usize { + inventory + .targets + .iter() + .position(|t| { + matches!( + inventory.aggregate(t).non_asap(), + Some(NonASAPOp::Aggregate { measures, .. }) + if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) + ) + }) + .expect("a top-k target") + } + + /// How many whole-workload choices enumeration keeps: every digit over + /// its target's `choice_count`, less those [`absorbed_choice_varies`] + /// skips. + /// + /// [`absorbed_choice_varies`]: PerComputationCandidates::absorbed_choice_varies + fn enumerated_choices(inventory: &PerComputationCandidates) -> usize { + let mut enumerated = 0; + let mut choice = vec![0; inventory.targets.len()]; + loop { + if !inventory.absorbed_choice_varies(&choice) { + enumerated += 1; + } + let mut carry = true; + for (digit, target) in choice.iter_mut().zip(&inventory.targets).rev() { + *digit += 1; + if *digit < target.choice_count() { + carry = false; + break; + } + *digit = 0; + } + if carry { + return enumerated; + } + } + } + + /// Only raw data samples and plain sums of them: arithmetic, `rate` and other + /// functions end the requirement. + #[test] + fn raw_data_samples_of_metric_ends_at_any_other_operator() { + for (query, metric) in [ + ("sum_over_time(m[1m])", Some("m")), + ("sum by (job) (sum_over_time(m[1m]))", Some("m")), + ("sum_over_time(m[1m]) - 100", None), + ("-sum_over_time(m[1m])", None), + ("rate(m[1m])", None), + ("max_over_time(m[1m])", None), + ] { + let root = crate::test_support::lower_promql(query, AccuracyTarget::exact()); + assert_eq!( + raw_data_samples_of_metric(&root).as_deref(), + metric, + "{query}" + ); + } + } + + /// The target whose fused computation is a nested-sum merge. + fn nested_sum_target(inventory: &PerComputationCandidates) -> Option { + let merge = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); + inventory.targets.iter().position(|t| { + t.fused_computation + .as_ref() + .is_some_and(|fused| fused.ways_to_compute == [merge.clone()]) + }) + } + + /// The merge needs a sum over a sum's value: an outer sum over a rate, + /// an outer count, or an outer sum over an inner `max` gets none. + #[test] + fn nested_sum_merge_needs_a_sum_of_sums() { + for query in [ + "sum by (job) (rate(m[5m]))", + "count by (job) (sum_over_time(m[5m]))", + "sum by (job) (max_over_time(m[5m]))", + ] { + assert_eq!(nested_sum_target(&inventory(query)), None, "{query}"); + } + } + + /// A per-series sum drops the metric name, but the raw data samples keep it: + /// grouping the merge by `__name__` would answer the metric's name where + /// the nested sums answer none, so no merge is offered. + #[test] + fn nested_sum_merge_never_groups_by_the_dropped_metric_name() { + let query = "sum by (__name__) (sum_over_time(m[5m]))"; + assert_eq!(nested_sum_target(&inventory(query)), None); + } + + /// Merges do not stack: in `sum(sum by (job) (sum_over_time(m[5m])))` + /// a target with a fused computation is never absorbed, and `combinations` + /// counts exactly the choices enumeration keeps. + #[test] + fn nested_sum_merges_do_not_stack() { + let inventory = inventory("sum(sum by (job) (sum_over_time(m[5m])))"); + let absorbed: Vec = absorbed_targets(&inventory.targets).collect(); + assert_eq!(absorbed.len(), 1, "one merge"); + for &a in &absorbed { + assert!( + inventory.targets[a].fused_computation.is_none(), + "an absorbed target has a fused computation" + ); + } + let enumerated = enumerated_choices(&inventory); + assert_eq!(inventory.combinations(), enumerated); + } + + /// The shapes a heap over the raw data samples cannot answer do not get one: + /// `bottomk` stays a sort and limit (no top-k target), and a grouped sum + /// `without` labels or by the metric name (which the per-series sum drops) + /// gets no heap that absorbs the per-series sum and reads the samples. + #[test] + fn grouped_range_sums_a_raw_data_samples_heap_cannot_rank_get_no_fused_heap() { + let is_topk = |inventory: &PerComputationCandidates, t: &ComputationTarget| { + matches!( + inventory.aggregate(t).non_asap(), + Some(NonASAPOp::Aggregate { measures, .. }) + if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) + ) + }; + let bottom = inventory("bottomk(3, sum by (region) (sum_over_time(m[1m])))"); + assert!( + !bottom.targets.iter().any(|t| is_topk(&bottom, t)), + "bottomk has no top-k target" + ); + for query in [ + "topk(3, sum without (instance) (sum_over_time(m[1m])))", + "topk(3, sum by (__name__) (sum_over_time(m[1m])))", + ] { + let inventory = inventory(query); + for target in inventory.targets.iter().filter(|t| is_topk(&inventory, t)) { + assert!( + target + .fused_computation + .as_ref() + .is_none_or(|f| f.absorbed_targets.len() == 1), + "{query}: {:?}", + target.fused_computation + ); + } + } + } + + /// A fused computation picked leaves every target beneath it uncomputed, + /// also one that an absorbed target's own merge absorbs: Pass 1 closes + /// `absorbed_targets` over those, so enumeration keeps only their first choice and + /// `combinations` still counts exactly the choices kept. Here the top-k + /// heap is made to absorb the grouped sum only, whose merge absorbs the + /// per-series sum. + #[test] + fn a_fused_computation_absorbs_what_its_absorbed_targets_merges_absorb() { + let mut inventory = inventory("topk(10, sum by (job) (sum_over_time(m[1m])))"); + let topk = topk_target(&inventory); + let fused = inventory.targets[topk] + .fused_computation + .as_mut() + .expect("a fused heap"); + let grouped = fused.absorbed_targets[0]; + fused.absorbed_targets.truncate(1); + let per_series = inventory.targets[grouped] + .fused_computation + .as_ref() + .expect("the grouped sum's merge") + .absorbed_targets[0]; + close_absorptions(&mut inventory.targets); + assert_eq!( + inventory.targets[topk] + .fused_computation + .as_ref() + .unwrap() + .absorbed_targets, + vec![grouped, per_series] + ); + assert_eq!(inventory.combinations(), enumerated_choices(&inventory)); + } + + /// `combinations` counts each choice once: picking a fused + /// whole-expression candidate with any but the first candidate of the + /// target it absorbs is the same candidate. An absorbed target with three + /// candidates (an approximate inner count: its accumulator, Count-Min and + /// UnivMon) gives 1 exact · 3 + 2 whole-expression = 5, not 3 · 3. + #[test] + fn combinations_skip_choices_a_whole_expression_candidate_absorbs() { + let inventory = inventory("topk by (job) (10, count_over_time(m[1m]))"); + let topk = topk_target(&inventory); + let target = &inventory.targets[topk]; + let Some(fused) = &target.fused_computation else { + panic!("a fused computation: {:?}", target.ways_to_compute) + }; + let inner = fused.absorbed_targets[0]; + let n = target.ways_to_compute.len(); + let whole = fused.ways_to_compute.len(); + let m = inventory.targets[inner].ways_to_compute.len(); + let enumerated = enumerated_choices(&inventory); + assert!(whole > 0 && m > 1, "{n} {whole} {m}"); + assert_eq!(inventory.combinations(), n * m + whole); + assert_eq!(inventory.combinations(), 5); + assert_eq!(inventory.combinations(), enumerated); + } + + /// A Count-Min estimate has a bound only over weights proven + /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. + #[test] + fn count_min_estimate_has_no_bound_over_an_open_requirement() { + let family = FieldDataType::Sketch(SketchAlgorithm::Cms); + let query = SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: None, + }; + let with = |weight_domain| SummaryUpdate { + item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + weight_domain, + }; + assert!(!estimate_has_bound( + &family, + &with(WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }), + &query + )); + assert!(!estimate_has_bound( + &family, + &with(WeightDomain::UnknownOrSigned), + &query + )); + assert!(estimate_has_bound( + &family, + &with(WeightDomain::NonNegative { + proof: NonNegativeWeightProof::CounterRawDataSamples + }), + &query + )); + } + + /// Classic HLL's failure probability is left symbolic in a source's + /// distinct bound only for one instance per group over that source's + /// own rows (#693's scope condition): not over a join of two sources, + /// not over computed rows, and not for one Hydra structure shared by + /// every group, whose distinct items the source's bound does not bound. + #[test] + fn classic_hll_confidence_needs_one_instance_over_a_sources_own_rows() { + let hll = FieldDataType::Sketch; + let input = |query| crate::test_support::lower_promql(query, AccuracyTarget::exact()); + let guarantee = |family: &FieldDataType, query| { + source_distinct_guarantee(family, &SketchStatistic::Cardinality, &input(query)) + }; + let own = hll(SketchAlgorithm::Hll); + let symbolic = guarantee(&own, "a").expect("a source's own rows"); + assert_eq!( + symbolic.spec.failure_probability, + ProbabilityExpr::Unknown { + statistic: asap_accuracy::guarantee::hll::classic_failure_statistic(15, 0.05, "a") + } + ); + for (query, over) in [("a * on() b", "a join"), ("a * 2", "computed rows")] { + assert!(guarantee(&own, query).is_none(), "{over}: {query}"); + } + let hydra = hll(SketchAlgorithm::HydraHll); + assert!(guarantee(&hydra, "a").is_none()); + } +} diff --git a/crates/logical-asap-optimization/src/pass1/computation_targets_tests.rs b/crates/logical-asap-optimization/src/pass1/computation_targets_tests.rs new file mode 100644 index 000000000..f5ef38117 --- /dev/null +++ b/crates/logical-asap-optimization/src/pass1/computation_targets_tests.rs @@ -0,0 +1,748 @@ +//! Frontend-to-Pass-1 acceptance: candidate discovery precedes empirical selection. +use crate::pass1::computation_targets::{ + computation_candidates_for_intent, generate_per_computation_candidates, + PerComputationCandidateError, +}; +use crate::stage0::memo_of_queries; +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::operator::arguments::{Reduction, Source}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot, ScalarExpr, +}; +use asap_types::summary_type::{ExactKind, SketchAlgorithm, SummaryUpdate}; + +use asap_types::stages::{HavingEntry, HavingPlacement, HowToComputeAggregateMeasure}; +use std::rc::Rc; + +fn candidates_for(intent: AggIntent) -> Vec { + computation_candidates_for_intent(&intent).candidates +} +fn algorithms(choices: &[HowToComputeAggregateMeasure]) -> Vec { + choices + .iter() + .filter_map(|choice| match choice { + HowToComputeAggregateMeasure::Summary(algorithm) => Some(*algorithm), + _ => None, + }) + .collect() +} +fn scan(nullable_value: bool) -> Rc { + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "flows".into(), + }, + predicates: vec![], + schema: Schema::lifted( + vec![ + Field::plain("src_ip", DataType::Utf8, false), + Field::plain("value", DataType::Float64, nullable_value), + ], + None, + ), + })) + .unwrap() +} +fn aggregate_over( + child: Rc, + measures: Vec, + filters: Vec>, + having: Option, +) -> Rc { + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { + child, + reduction: Reduction::by(vec![]), + measures, + output_names: vec![], + filters, + having, + })) + .unwrap() +} +fn aggregate(intent: AggIntent) -> Rc { + aggregate_over(scan(false), vec![intent], vec![], None) +} +fn always() -> Predicate { + Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))) +} + +/// Example 2 preserves specialized distinct summaries and the universal alternative. +#[test] +fn cardinality_keeps_exact_specialized_and_universal_candidates() { + let choices = candidates_for(AggIntent::Cardinality { cols: vec![0] }); + assert_eq!(choices[0], HowToComputeAggregateMeasure::Exact); + assert_eq!( + algorithms(&choices), + vec![ + SketchAlgorithm::Hll, + SketchAlgorithm::Kmv, + SketchAlgorithm::UnivMon + ] + ); + let tuple = candidates_for(AggIntent::Cardinality { cols: vec![0, 1] }); + assert!(!algorithms(&tuple).contains(&SketchAlgorithm::UnivMon)); +} + +/// Frequency moments retain exact execution and a universal sketch without certification. +#[test] +fn frequency_statistics_keep_universal_choices() { + for intent in [ + AggIntent::FrequencyL2 { col: Some(0) }, + AggIntent::FrequencyEntropy { col: Some(0) }, + ] { + let choices = candidates_for(intent); + assert_eq!(choices[0], HowToComputeAggregateMeasure::Exact); + assert_eq!(algorithms(&choices), vec![SketchAlgorithm::UnivMon]); + } +} + +/// Scalar roots expose their producer targets; repeated references retain one target identity. +#[test] +fn scalar_root_producers_are_discovered_once() { + let producer = aggregate(AggIntent::Cardinality { cols: vec![0] }); + let roots = vec![ + QueryRoot::Scalar(ScalarExpr::ScalarSubquery(producer.clone())), + QueryRoot::Operator(producer.clone()), + ]; + let candidates = generate_per_computation_candidates( + &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), + ) + .unwrap(); + assert_eq!(candidates.roots.len(), 2); + for root in &candidates.roots { + root.validate_structure().unwrap(); + } + assert_eq!(candidates.targets.len(), 1); + assert_eq!(**candidates.aggregate(&candidates.targets[0]), *producer); + + assert!(producer.guarantee.is_none()); + producer.validate_structure().unwrap(); +} + +/// #509 Example 1 Q1 is exact: each target's one exact option is its exact +/// accumulator state (grouped sum, rate); no plain exact form and no summary. +/// One PromQL query with the default exact target, lowered through the +/// workload frontend. +fn exact_promql(query: &str) -> Vec { + use asap_types::workload::{ + BatchEntry, PlanningWorkload, Query, QueryLanguage, QueryRequirements, QueryWorkload, + }; + let workload = PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![BatchEntry { + query: Query(query.into()), + requirements: QueryRequirements { + accuracy: AccuracyTarget::exact(), + ..Default::default() + }, + predictability: Default::default(), + invocations: 1, + execute_at: None, + range_evaluation: None, + }]), + repeating_queries: None, + }, + data_workload: Some(asap_types::workload::DataWorkload::sampled_every( + asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), + asap_types::workload::DurationMs(1000), + )), + }; + asap_frontend_promql::lower_promql_query_workload(&workload).unwrap() +} + +#[test] +fn example_1_exact_q1_keeps_exact_accumulators_only() { + let roots = exact_promql("sum by (job) (rate(http_requests_total[1m]))"); + let candidates = generate_per_computation_candidates( + &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), + ) + .unwrap(); + let found: Vec<_> = candidates + .targets + .iter() + .map(|target| target.ways_to_compute.clone()) + .collect(); + assert_eq!( + found, + vec![ + vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ + ExactKind::Sum + ])], + vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ + ExactKind::Rate + ])], + ] + ); +} + +/// ASAP operators fail at the stage +/// boundary. +#[test] +fn asap_input_is_rejected() { + let summary = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + child: scan(false), + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::by(vec![]), + filter: None, + })) + .unwrap(); + assert!(matches!( + generate_per_computation_candidates( + &memo_of_queries( + &[QueryRoot::Operator(summary)], + &[None], + &Default::default() + ) + .unwrap() + ), + Err(PerComputationCandidateError::ContainsAsap) + )); +} + +/// TopK keeps both declared heap substrates without choosing an implementation. +#[test] +fn topk_keeps_both_specialized_heap_choices() { + let choices = candidates_for(AggIntent::TopK { k: 10 }); + assert_eq!(choices[0], HowToComputeAggregateMeasure::Exact); + assert_eq!( + algorithms(&choices), + vec![ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap + ] + ); +} + +/// Summary candidates are unsized, so they do not depend on the requested error. +#[test] +fn summary_candidates_are_unsized() { + let loose = candidates_for(AggIntent::Count); + let tight = candidates_for(AggIntent::Count); + assert_eq!(loose, tight); +} + +/// Each measure of a multi-measure aggregate is its own target; a measure whose +/// only realization is its exact accumulator is still a target. +#[test] +fn multi_measure_aggregate_has_one_target_per_measure() { + let node = aggregate_over( + scan(false), + vec![AggIntent::Sum { col: Some(1) }, AggIntent::Count], + vec![], + None, + ); + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(node)], &[None], &Default::default()).unwrap(), + ) + .unwrap(); + let candidates = &inventory.targets; + assert_eq!( + candidates + .iter() + .map(|target| target.measure_index) + .collect::>(), + vec![0, 1] + ); + assert_eq!( + candidates[0].ways_to_compute, + vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ + ExactKind::Sum + ])] + ); + assert_sql_count_choices(&choices_of(&inventory, 1)); +} + +/// An aggregate with no realization besides exact execution is not a target. +#[test] +fn aggregate_without_realization_is_not_a_target() { + let node = aggregate(AggIntent::StdDev { + col: Some(1), + population: true, + }); + assert!(generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(node)], &[None], &Default::default()).unwrap() + ) + .unwrap() + .targets + .is_empty()); +} + +/// A filtered SQL count keeps its executable accumulator and records its +/// filter, which composition applies to the state. +#[test] +fn filtered_measure_keeps_candidates_and_records_its_filter() { + let count = || AggIntent::Count; + let filtered = aggregate_over( + scan(false), + vec![count(), count()], + vec![Some(always()), None], + None, + ); + let inventory = generate_per_computation_candidates( + &memo_of_queries( + &[QueryRoot::Operator(filtered)], + &[None], + &Default::default(), + ) + .unwrap(), + ) + .unwrap(); + let targets = &inventory.targets; + assert_eq!(targets.len(), 2); + for (index, target) in targets.iter().enumerate() { + assert_sql_count_choices(&choices_of(&inventory, index)); + assert!(target.having_filters.is_empty()); + } + assert_eq!(targets[0].measure_filter, Some(always())); + assert_eq!(targets[1].measure_filter, None); +} + +/// Target `target`'s own ways to compute. +fn choices_of( + inventory: &crate::pass1::computation_targets::PerComputationCandidates, + target: usize, +) -> Vec { + inventory.targets[target].ways_to_compute.clone() +} + +fn assert_sql_count_choices(choices: &[HowToComputeAggregateMeasure]) { + assert_eq!( + choices, + vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ + ExactKind::Count + ])] + ); +} + +/// `count(*) by (src_ip)`, the form the SQL frontend gives `GROUP BY src_ip`. +fn grouped_count(having: Option) -> Rc { + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { + child: scan(false), + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Count], + output_names: vec![], + filters: vec![], + having, + })) + .unwrap() +} +/// `HAVING` as the SQL frontend lowers it: a `Filter` above the aggregate. +fn having_filter(child: Rc, pred: Predicate) -> Rc { + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { pred, child })).unwrap() +} +/// `column = 'a'` over the aggregate output (0: `src_ip`, 1: the count). +fn equals(column: usize, value: ScalarValue) -> Predicate { + Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(value)), + semantics: ExprSemantics::Sql, + }) +} + +/// A `HAVING` over group keys only is pushed below the aggregate and keeps +/// every executable candidate. +#[test] +fn key_only_having_is_pushed_below_and_keeps_candidates() { + let node = grouped_count(None); + let root = having_filter(node.clone(), equals(0, ScalarValue::Utf8("a".into()))); + let inventory = generate_per_computation_candidates( + &memo_of_queries( + &[QueryRoot::Operator(root.clone())], + &[None], + &Default::default(), + ) + .unwrap(), + ) + .unwrap(); + let targets = &inventory.targets; + assert_eq!(targets.len(), 1); + assert_eq!(**inventory.aggregate(&targets[0]), *node); + assert_eq!( + targets[0].having_filters, + vec![HavingEntry { + filter: Some(root.clone()), + predicate: equals(0, ScalarValue::Utf8("a".into())), + placement: HavingPlacement::BelowAggregate, + }] + ); + assert_sql_count_choices(&choices_of(&inventory, 0)); +} + +/// A `HAVING` over an aggregate value, as a `Filter` above or as the +/// aggregate's own `having`, stays above the output and keeps executable candidates. +#[test] +fn value_having_stays_above_output_and_keeps_candidates() { + let value = || equals(1, ScalarValue::Int64(10)); + let inventory = generate_per_computation_candidates( + &memo_of_queries( + &[ + QueryRoot::Operator(having_filter(grouped_count(None), value())), + QueryRoot::Operator(grouped_count(Some(value()))), + ], + &vec![None; 2], + &Default::default(), + ) + .unwrap(), + ) + .unwrap(); + let targets = &inventory.targets; + assert_eq!(targets.len(), 2); + for (index, target) in targets.iter().enumerate() { + let placements: Vec<_> = target + .having_filters + .iter() + .map(|entry| entry.placement) + .collect(); + assert_eq!(placements, vec![HavingPlacement::AboveOutput]); + assert_sql_count_choices(&choices_of(&inventory, index)); + } +} + +/// A key-only `HAVING` above a shared aggregate stays above the output when +/// another consumer reads the aggregate without it. +#[test] +fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { + let node = grouped_count(None); + let key = equals(0, ScalarValue::Utf8("a".into())); + let filtered = having_filter(node.clone(), key.clone()); + let targets = generate_per_computation_candidates( + &memo_of_queries( + &[ + QueryRoot::Operator(filtered.clone()), + QueryRoot::Operator(node.clone()), + ], + &vec![None; 2], + &Default::default(), + ) + .unwrap(), + ) + .unwrap() + .targets; + assert_eq!(targets.len(), 1); + assert_eq!( + targets[0].having_filters, + vec![HavingEntry { + filter: Some(filtered), + predicate: key.clone(), + placement: HavingPlacement::AboveOutput, + }] + ); + + // Two consumers applying the same key-only predicate push it below. + let first = having_filter(node.clone(), key.clone()); + let second = having_filter(node, key); + let targets = generate_per_computation_candidates( + &memo_of_queries( + &[QueryRoot::Operator(first), QueryRoot::Operator(second)], + &[None, None], + &Default::default(), + ) + .unwrap(), + ) + .unwrap() + .targets; + assert!(targets[0] + .having_filters + .iter() + .all(|entry| entry.placement == HavingPlacement::BelowAggregate)); + // The two equal `Filter`s are one memo group, so one HAVING entry. + assert_eq!(targets[0].having_filters.len(), 1); +} + +/// A scalar reference to a grouped aggregate (`Filter(scan, EXISTS G)` as a +/// second root) is a consumer that reads it without the `HAVING`, so the +/// first root's key-only `HAVING` stays above the output. +#[test] +fn a_scalar_reference_consumer_keeps_a_key_only_having_above() { + let node = grouped_count(None); + let key = equals(0, ScalarValue::Utf8("a".into())); + let filtered = having_filter(node.clone(), key.clone()); + let exists = having_filter( + scan(false), + Predicate(ScalarExpr::Exists { + subquery: node.clone(), + negated: false, + }), + ); + let inventory = generate_per_computation_candidates( + &memo_of_queries( + &[ + QueryRoot::Operator(filtered.clone()), + QueryRoot::Operator(exists), + ], + &vec![None; 2], + &Default::default(), + ) + .unwrap(), + ) + .unwrap(); + let targets = &inventory.targets; + let target = targets + .iter() + .find(|target| **inventory.aggregate(target) == *node) + .expect("the grouped count's target"); + assert_eq!( + target.having_filters, + vec![HavingEntry { + filter: Some(filtered), + predicate: key, + placement: HavingPlacement::AboveOutput, + }] + ); +} + +/// Of two stacked `Filter`s above an aggregate, only the adjacent one is +/// its `HAVING`; the outer one filters the inner `Filter`'s output. +#[test] +fn only_the_adjacent_of_stacked_filters_is_having() { + let node = grouped_count(None); + let inner_key = equals(0, ScalarValue::Utf8("a".into())); + let inner = having_filter(node.clone(), inner_key.clone()); + let outer = having_filter(inner.clone(), equals(0, ScalarValue::Utf8("b".into()))); + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(outer)], &[None], &Default::default()).unwrap(), + ) + .unwrap() + .targets; + assert_eq!(targets.len(), 1); + assert_eq!( + targets[0].having_filters, + vec![HavingEntry { + filter: Some(inner), + predicate: inner_key, + placement: HavingPlacement::BelowAggregate, + }] + ); +} + +/// A global aggregate's `HAVING` always stays above the output, even a +/// constant one: the one output row exists with no input rows. +#[test] +fn global_aggregate_having_stays_above() { + let count = || vec![AggIntent::Count]; + let never = Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))); + let targets = generate_per_computation_candidates( + &memo_of_queries( + &[ + QueryRoot::Operator(having_filter( + aggregate_over(scan(false), count(), vec![], None), + never.clone(), + )), + QueryRoot::Operator(aggregate_over(scan(false), count(), vec![], Some(never))), + ], + &vec![None; 2], + &Default::default(), + ) + .unwrap(), + ) + .unwrap() + .targets; + assert_eq!(targets.len(), 2); + for target in &targets { + let placements: Vec<_> = target + .having_filters + .iter() + .map(|entry| entry.placement) + .collect(); + assert_eq!(placements, vec![HavingPlacement::AboveOutput]); + } +} + +/// The `HAVING` entries of the target whose measure satisfies `is`. +fn having_of( + inventory: &crate::pass1::computation_targets::PerComputationCandidates, + is: impl Fn(&AggIntent) -> bool, +) -> Vec { + let target = inventory + .targets + .iter() + .find(|target| match inventory.aggregate(target).non_asap() { + Some(NonASAPOp::Aggregate { measures, .. }) => is(&measures[target.measure_index]), + _ => false, + }) + .expect("target"); + target.having_filters.clone() +} + +/// `Column(column) > ScalarSubquery(subquery)`. +fn above_scalar(column: usize, subquery: Rc) -> Predicate { + Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::ScalarSubquery(subquery)), + semantics: ExprSemantics::Sql, + }) +} + +/// A `Filter` that reads an aggregate only from its predicate is not that +/// aggregate's `HAVING`; one that also filters the aggregate gives one entry. +#[test] +fn scalar_reference_filter_is_not_having() { + let count = || vec![AggIntent::Count]; + let agg = aggregate_over(scan(false), count(), vec![], None); + let root = having_filter(scan(false), above_scalar(1, agg.clone())); + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(root)], &[None], &Default::default()).unwrap(), + ) + .unwrap(); + let targets = &inventory.targets; + assert_eq!(targets.len(), 1); + assert_eq!(**inventory.aggregate(&targets[0]), *agg); + assert!( + targets[0].having_filters.is_empty(), + "{:?}", + targets[0].having_filters + ); + + let agg = aggregate_over(scan(false), count(), vec![], None); + let root = having_filter(agg.clone(), above_scalar(0, agg.clone())); + let targets = generate_per_computation_candidates( + &memo_of_queries( + &[QueryRoot::Operator(root.clone())], + &[None], + &Default::default(), + ) + .unwrap(), + ) + .unwrap() + .targets; + assert_eq!(targets.len(), 1); + assert_eq!( + targets[0].having_filters, + vec![HavingEntry { + filter: Some(root), + predicate: above_scalar(0, agg), + placement: HavingPlacement::AboveOutput, + }] + ); +} + +/// PromQL `scalar(...)` operands are scalar references, not `HAVING`. +#[test] +fn promql_scalar_operand_is_not_having() { + let roots = exact_promql("http_requests_total > scalar(sum(http_requests_total))"); + let targets = generate_per_computation_candidates( + &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), + ) + .unwrap() + .targets; + assert!(!targets.is_empty()); + for target in &targets { + assert!( + target.having_filters.is_empty(), + "{:?}", + target.having_filters + ); + } + + let roots = exact_promql("sum by (job) (x) > scalar(count(x))"); + let inventory = generate_per_computation_candidates( + &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), + ) + .unwrap(); + assert!(having_of(&inventory, |intent| matches!(intent, AggIntent::Count)).is_empty()); + let sum = having_of(&inventory, |intent| matches!(intent, AggIntent::Sum { .. })); + assert_eq!(sum.len(), 1, "{sum:?}"); + assert_eq!(sum[0].placement, HavingPlacement::AboveOutput); +} + +/// A non-root, non-`Filter` consumer keeps a key-only `HAVING` above. +#[test] +fn having_shared_with_a_project_consumer_stays_above() { + let node = grouped_count(None); + let key = equals(0, ScalarValue::Utf8("a".into())); + let filtered = having_filter(node.clone(), key); + let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { + cols: vec![asap_types::ir::ProjectItem { + alias: None, + expr: ScalarExpr::Column(0), + }], + qualifier: None, + child: node, + })) + .unwrap(); + let targets = generate_per_computation_candidates( + &memo_of_queries( + &[QueryRoot::Operator(filtered), QueryRoot::Operator(project)], + &[None, None], + &Default::default(), + ) + .unwrap(), + ) + .unwrap() + .targets; + assert_eq!(targets.len(), 1); + let placements: Vec<_> = targets[0] + .having_filters + .iter() + .map(|entry| entry.placement) + .collect(); + assert_eq!(placements, vec![HavingPlacement::AboveOutput]); +} + +/// `without(...)` and per-entity aggregates have no fixed key prefix, so a +/// `HAVING` over column 0 stays above. +#[test] +fn without_and_per_entity_having_stays_above() { + for reduction in [ + Reduction::Reduce(asap_types::ir::operator::arguments::GroupKeys::without( + vec![1], + )), + Reduction::PerEntity, + ] { + let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { + child: scan(false), + reduction: reduction.clone(), + measures: vec![AggIntent::Count], + output_names: vec![], + filters: vec![], + having: None, + })) + .unwrap(); + let root = having_filter(node, equals(0, ScalarValue::Utf8("a".into()))); + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(root)], &[None], &Default::default()).unwrap(), + ) + .unwrap() + .targets; + assert_eq!(targets.len(), 1, "{reduction:?}"); + let placements: Vec<_> = targets[0] + .having_filters + .iter() + .map(|entry| entry.placement) + .collect(); + assert_eq!( + placements, + vec![HavingPlacement::AboveOutput], + "{reduction:?}" + ); + } +} + +// Structurally equal aggregates are one memo group with one target, and its +// HAVING placement is decided over every consumer of the group: an unfiltered +// consumer keeps the key-only HAVING above the output for the filtered ones. +#[test] +fn memo_cached_rules_preserve_occurrence_specific_having() { + let shared = grouped_count(None); + let other = grouped_count(None); + let key = equals(0, ScalarValue::Utf8("a".into())); + let roots = vec![ + QueryRoot::Operator(having_filter(shared.clone(), key.clone())), + QueryRoot::Operator(shared), + QueryRoot::Operator(having_filter(other, key)), + ]; + let memo = memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(); + let cached = generate_per_computation_candidates(&memo).unwrap(); + assert_eq!(cached.targets.len(), 1); + // Roots 0 and 2 apply the same `Filter`, one memo group: one entry. + let [entry] = &cached.targets[0].having_filters[..] else { + panic!("one HAVING entry: {:?}", cached.targets[0].having_filters); + }; + assert_eq!(entry.placement, HavingPlacement::AboveOutput); +} diff --git a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs b/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs deleted file mode 100644 index 1d20e5e79..000000000 --- a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs +++ /dev/null @@ -1,310 +0,0 @@ -//! Stage 1, Pass 1's entry and output: the per-computation candidates attached to the -//! memo groups of the workload's hash-consed DAG -//! ([`PerComputationLogicalMemoCandidates`]), which Pass 2 extends. - -use std::rc::Rc; - -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::{QueryRoot, SchemaDerivationError}; -use asap_types::workload::DataWorkload; - -use asap_accuracy::sizing::{ - per_input_failure_probability, with_summaries_sized_for_failure_probability, -}; -use asap_accuracy::{AccuracyEvidenceProvider, AccuracyModel}; - -use super::memo_population::try_populate_pass1; -use super::memo_population::CandidatePopulationError; -use crate::deployment_capabilities::SupportedSummary; -use crate::pass1::per_computation_candidates::{ - generate_per_computation_candidates, PerComputationCandidateError, -}; -use asap_types::stages::{ - DagOfLogicalMemoGroups, Memo, PerComputationLogicalMemoCandidates, PrunedCandidate, -}; - -/// Why [`generate_per_computation_logical_memo_candidates`] failed. -#[derive(Debug, thiserror::Error)] -pub enum PerComputationLogicalMemoCandidatesError { - /// A query's DAG cannot be planned: it already holds ASAP or physical - /// annotations, or a measure's accuracy target is invalid. - #[error(transparent)] - PerComputation(#[from] PerComputationCandidateError), - /// The targets do not belong to the DAG of logical memo groups. - #[error(transparent)] - Population(#[from] CandidatePopulationError), - /// A candidate's summaries sized for its readers' failure probability - /// did not rebuild. - #[error(transparent)] - Sizing(#[from] SchemaDerivationError), -} - -/// Stage 1, Pass 1: the per-computation candidates on the memo groups of -/// `dag_of_logical_memo_groups`. It finds one computation target per -/// measure of each aggregate group with a realization besides exact -/// execution, drops the summaries the deployment does not support -/// (`supported_summaries`, the deployment's -/// `DeploymentCapabilities::supported_summaries`; `None` supports every -/// one) and, unless `accuracy` or `evidence` may accept an output without -/// its guarantee, the summaries no query reading them can use (a query -/// `q`'s target is `accuracy_targets[q]`). It then attaches each group's -/// candidates, each with its summaries sized for its readers' failure -/// probabilities (`size_for_failure_probability`): one sized candidate -/// per accuracy-budget split. Every pruned way to compute a measure is in -/// the output's `pruned_candidates`. -pub fn generate_per_computation_logical_memo_candidates( - dag_of_logical_memo_groups: DagOfLogicalMemoGroups, - accuracy_targets: &[AccuracyTarget], - accuracy: &dyn AccuracyModel, - evidence: &dyn AccuracyEvidenceProvider, - data: &DataWorkload, - supported_summaries: Option<&[SupportedSummary]>, -) -> Result { - let mut per_computation = generate_per_computation_candidates(&dag_of_logical_memo_groups)? - .without_unsupported_summaries(supported_summaries); - if !accuracy.may_accept_regardless_of_guarantee() && !evidence.may_provide_evidence() { - per_computation = - per_computation.without_summaries_no_query_can_use(accuracy_targets, data, accuracy); - } - let mut memo = try_populate_pass1(dag_of_logical_memo_groups, &per_computation)?; - size_for_failure_probability(&mut memo, &per_computation.roots, accuracy_targets, data)?; - let pruned_candidates = per_computation - .pruned_ways_to_compute - .into_iter() - .map(|pruned| PrunedCandidate { - group: per_computation.targets[pruned.target].aggregate_group, - candidate: pruned.way_to_compute, - reason: pruned.reason, - }) - .collect(); - Ok(PerComputationLogicalMemoCandidates { - memo, - pruned_candidates, - }) -} - -/// Sizes every candidate of `memo` for its readers' failure probabilities, -/// in place: each approximate input of a candidate with `n` -/// accuracy budget parts (`accuracy_budget_parts`) gets the smallest -/// [`per_input_failure_probability`] `δ_q / (n · V_q)` among the queries `q` -/// reaching its group, `V_q` the values `q`'s target covers (`G` of its -/// root, `roots[q]`, for the whole result) -/// ([`with_summaries_sized_for_failure_probability`]). These are the candidates' final -/// sizes: Pass 2 inherits them or takes the strictest shared reader's -/// ([`size_for_all_readers`](asap_accuracy::estimators::size_for_all_readers)), -/// and Stage 3 only checks the composed accuracy. A candidate no reader -/// gives a failure probability (exact, per-value with no `δ`, or a -/// whole-result target over an unbound `G`) keeps its Pass 1 size. -fn size_for_failure_probability( - memo: &mut Memo, - roots: &[QueryRoot], - accuracy_targets: &[AccuracyTarget], - data: &DataWorkload, -) -> Result<(), PerComputationLogicalMemoCandidatesError> { - for (group, candidates) in memo.groups.iter().zip(&mut memo.candidates) { - for candidate in candidates.iter_mut() { - let Some(failure_probability) = group - .reached_by_queries() - .iter() - .filter_map(|&query| { - per_input_failure_probability( - accuracy_targets.get(query)?, - candidate.accuracy_budget_parts, - roots.get(query)?, - data, - ) - }) - .reduce(f64::min) - else { - continue; - }; - let sized = with_summaries_sized_for_failure_probability( - &[QueryRoot::Operator(Rc::clone(&candidate.root_node))], - failure_probability, - data, - )?; - let [QueryRoot::Operator(node)] = &sized[..] else { - unreachable!("one operator root in, one out") - }; - candidate.root_node = Rc::clone(node); - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{ - lower_promql, shared_summary_state_groups, sharing_rules_and_shared_state_groups, - }; - use asap_accuracy::{IgnoreAccuracyGuarantees, NoAccuracyEvidence}; - use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; - use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; - use asap_types::ir::{ASAPOp, OperatorNode}; - use asap_types::stages::CandidateForMemoGroup; - use asap_types::workload::DurationMs; - - /// Rank error 0.01 at δ = 0.01 over `scope`. - fn rank_target(scope: AccuracyScope) -> AccuracyTarget { - AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(scope) - } - - /// Metric `x` with `series` series (`None`: undeclared). - fn data(series: Option) -> DataWorkload { - let data = DataWorkload::sampled_every(["x".to_string()], DurationMs(1_000)); - match series { - Some(series) => data.with_series_count(series), - None => data, - } - } - - /// Pass 1's memo for `queries`, each lowered and entered with its target. - fn pass1(queries: &[(&str, AccuracyTarget)], data: &DataWorkload) -> Memo { - let roots: Vec = queries - .iter() - .map(|(query, target)| { - QueryRoot::Operator(lower_promql( - query, - target.clone().accuracy_scope(AccuracyScope::PerValue), - )) - }) - .collect(); - let targets: Vec = - queries.iter().map(|(_, target)| target.clone()).collect(); - generate_per_computation_logical_memo_candidates( - generate_dag_of_logical_memo_groups(&roots), - &targets, - &IgnoreAccuracyGuarantees, - &NoAccuracyEvidence, - data, - None, - ) - .unwrap() - .memo - } - - /// The failure probability of every KLL `candidates` build. - fn kll_failure_probabilities<'a>( - candidates: impl IntoIterator, - ) -> Vec> { - candidates - .into_iter() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::Kll => match kind.params() { - SketchParams::Kll { - failure_probability, - .. - } => Some(*failure_probability), - _ => None, - }, - _ => None, - }) - .collect() - } - - /// The failure probability of every KLL build of `memo`'s candidates - /// for one accuracy budget part (the whole budget). - fn whole_share_kll(memo: &Memo) -> Vec> { - kll_failure_probabilities( - memo.candidates - .iter() - .flatten() - .filter(|candidate| candidate.accuracy_budget_parts == 1), - ) - } - - /// Pass 1 sizes each sketch for `δ / V`, `V` the values its query's - /// target covers: one for a per-value target, `⌊1 / share⌋` = 20 for - /// the groups holding at least 5% of the records (whether or not `G` is - /// declared), `G` = 10 000 series for the whole result. A whole-result - /// target over an undeclared `G` leaves the per-value size (KLL's - /// default failure probability, `None`), which Stage 3 then rejects. - #[test] - fn pass1_sizes_each_sketch_for_the_values_its_target_covers() { - let query = "quantile_over_time(0.99, x[5m])"; - let five_percent = AccuracyScope::GroupsWithRecordShareAtLeast(0.05); - for (scope, series, failure_probability) in [ - (AccuracyScope::PerValue, Some(10_000), None), - (five_percent, None, Some(0.01 / 20.0)), - (five_percent, Some(10_000), Some(0.01 / 20.0)), - ( - AccuracyScope::WholeResult, - Some(10_000), - Some(0.01 / 10_000.0), - ), - (AccuracyScope::WholeResult, None, None), - ] { - let memo = pass1(&[(query, rank_target(scope))], &data(series)); - let found = whole_share_kll(&memo); - assert!(!found.is_empty(), "{scope:?}"); - assert!( - found.iter().all(|f| *f == failure_probability), - "{scope:?} {series:?}: {found:?}" - ); - } - } - - /// A group several queries reach is sized for the strictest of them: - /// the per-series quantiles are both a whole-result query's value (over - /// `G` = 10 000 series) and the input of a global `max` (one value), so - /// they take `δ / 10 000`. - #[test] - fn a_group_several_queries_reach_takes_the_strictest_failure_probability() { - let whole = rank_target(AccuracyScope::WholeResult); - let memo = pass1( - &[ - ("quantile_over_time(0.99, x[5m])", whole.clone()), - ("max(quantile_over_time(0.99, x[5m]))", whole), - ], - &data(Some(10_000)), - ); - let found = whole_share_kll(&memo); - assert!(!found.is_empty()); - assert!( - found.iter().all(|f| *f == Some(0.01 / 10_000.0)), - "{found:?}" - ); - } - - /// Pass 2 shares one build among readers sized differently in Pass 1 - /// at the strictest reader's size: a per-value 0.95 quantile (δ) and a - /// whole-result 0.99 quantile over `G` = 10 000 series (δ / 10 000) - /// share a KLL sized for δ / 10 000. Stage 3 does not resize it. - #[test] - fn a_shared_build_takes_its_strictest_readers_size() { - let memo = pass1( - &[ - ( - "quantile_over_time(0.95, x[5m])", - rank_target(AccuracyScope::PerValue), - ), - ( - "quantile_over_time(0.99, x[5m])", - rank_target(AccuracyScope::WholeResult), - ), - ], - &data(Some(10_000)), - ); - let memo = sharing_rules_and_shared_state_groups(memo).unwrap(); - let found = kll_failure_probabilities( - shared_summary_state_groups(&memo) - .iter() - .filter(|group| memo.groups[group.0 as usize].reached_by_queries() == [0, 1]) - .flat_map(|group| &memo.candidates[group.0 as usize]), - ); - assert!(!found.is_empty()); - assert!( - found.iter().all(|f| *f == Some(0.01 / 10_000.0)), - "{found:?}" - ); - } -} diff --git a/crates/logical-asap-optimization/src/pass1/memo_population.rs b/crates/logical-asap-optimization/src/pass1/memo_population.rs deleted file mode 100644 index b3b411f1a..000000000 --- a/crates/logical-asap-optimization/src/pass1/memo_population.rs +++ /dev/null @@ -1,1231 +0,0 @@ -//! The hash-consed group DAG and Pass 1's implementation candidates per -//! group (design steps 1 and 2, Pass 1 part). -//! -//! Groups are the workload's logical DAG interned by -//! [`crate::stage0::cse::share_common_sub_dags`], the identical-expression rule's legality -//! included: structurally equal nodes are one group when sharing them is -//! legal: an ASAP node, or a non-ASAP node that returns the same rows for -//! the same input rows (`NonASAPOp::is_shareable_when_equal`). -//! A unique key is not needed; an unordered `LIMIT` stays one group per -//! reader. -//! -//! A Pass 1 target's aggregate group gets one candidate per combination of -//! its measures' Pass 1 candidates: the aggregate alone composed by -//! [`super::per_computation_candidates::compose_logical_candidate`] with that choice, every target beneath it -//! exact. Its local nodes are the composed nodes above the first node -//! equal to one of the group's descendant groups, which are its children. -//! The local nodes read each child group through its node (the same -//! `Rc`), and the candidate outputs the group's schema, so a parent reads -//! it in the group's place (a query's top-k root, which no parent reads, -//! may output the ranked value beside the key). -//! -//! A fused computation is one more candidate of the outer group, whatever -//! else reads the groups it absorbs: those keep their own candidates for -//! their other readers. A candidate equal to one the group already has (the -//! same structural id, input groups, fused groups and accuracy budget parts) is not -//! added again. -//! -//! A query whose value combines `n` approximate inputs splits its budget -//! into `n` equal shares, so a summary's size depends on its readers: each -//! sketch candidate is offered once per share `1..=n`, `n` the most -//! approximate inputs a query reaching the group can have -//! ([`CandidateForMemoGroup::accuracy_budget_parts`]). This records the sizing -//! configuration used during generation; the concrete sketch parameters carry -//! the resulting error contract. The connection planner may select a conservative -//! larger configuration, and validates the recomposed query-root guarantee. - -use asap_types::stages::HowToComputeAggregateMeasure; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use super::per_computation_candidates::{ - compose_logical_candidate_with_accuracy_budget_parts, PerComputationCandidates, -}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; - -use asap_types::stages::{ - CandidateForMemoGroup, DagOfLogicalMemoGroups, Memo, MemoGroup, MemoGroupId, - OriginOfCandidateForMemoGroup, -}; - -/// A Pass 1 inventory does not belong to the initial logical memo. -#[derive(Debug, thiserror::Error, PartialEq, Eq)] -pub enum CandidatePopulationError { - #[error("Pass 1 root dependencies differ from the initial logical memo")] - RootDependenciesChanged, - #[error("Pass 1 target {target} is not a valid measure of an initial aggregate group")] - InvalidTarget { target: usize }, - #[error("Pass 1 target {target} references missing absorbed target {absorbed}")] - InvalidAbsorption { target: usize, absorbed: usize }, -} - -/// Populate implementation candidates without changing initial group identities. -/// The inventory must have walked `initial`'s groups -/// (`PerComputationCandidates::walked`); its targets name them by id. -pub fn try_populate_pass1( - initial: DagOfLogicalMemoGroups, - per_computation: &PerComputationCandidates, -) -> Result { - if !per_computation.walked(&initial) { - return Err(CandidatePopulationError::RootDependenciesChanged); - } - for (index, target) in per_computation.targets.iter().enumerate() { - if !matches!( - per_computation.aggregate(target).non_asap(), - Some(asap_types::ir::NonASAPOp::Aggregate { measures, .. }) if target.measure_index < measures.len() - ) { - return Err(CandidatePopulationError::InvalidTarget { target: index }); - } - if let Some(fused) = &target.fused_computation { - if let Some(&absorbed) = fused - .absorbed_targets - .iter() - .find(|&&absorbed| absorbed >= per_computation.targets.len()) - { - return Err(CandidatePopulationError::InvalidAbsorption { - target: index, - absorbed, - }); - } - } - } - let candidates = pass1_candidates(per_computation, &initial.groups); - Ok(Memo { - groups: initial.groups, - query_roots: initial.query_roots, - candidates, - }) -} - -/// `g` and every group below it. -fn reach(groups: &[MemoGroup], g: MemoGroupId, reached: &mut HashSet) { - if reached.insert(g) { - for &child in &groups[g.0 as usize].child_groups { - reach(groups, child, reached); - } - } -} - -/// Every group's Pass 1 candidates, by group index. -fn pass1_candidates( - per_computation: &PerComputationCandidates, - groups: &[MemoGroup], -) -> Vec> { - // Each target aggregate group's targets, one per measure: Pass 1 walks - // the groups, so an aggregate several queries read has one target per - // measure. - let mut targets_of: HashMap> = HashMap::new(); - for (t, target) in per_computation.targets.iter().enumerate() { - targets_of - .entry(target.aggregate_group) - .or_default() - .push(t); - } - // The groups' own nodes, which the composed candidates read. - let logical_nodes: Rc<[Rc]> = groups - .iter() - .map(|group| Rc::clone(&group.logical_node)) - .collect(); - let own = |g: usize| CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: Rc::clone(&groups[g].logical_node), - input_groups: groups[g].child_groups.clone(), - origin: OriginOfCandidateForMemoGroup::Pass1, - accuracy_budget_parts: 1, - }; - // Each query's most approximate inputs: its targets that offer a - // summary, each one sketch state at most. - let offers_a_summary = |t: usize| { - let target = &per_computation.targets[t]; - let summary = |c: &HowToComputeAggregateMeasure| { - matches!(c, HowToComputeAggregateMeasure::Summary(_)) - }; - target.ways_to_compute.iter().any(summary) - || target - .fused_computation - .as_ref() - .is_some_and(|fused| fused.ways_to_compute.iter().any(summary)) - }; - let mut most_approximate_inputs = vec![0usize; per_computation.roots.len()]; - for (g, targets) in &targets_of { - let summaries = targets.iter().filter(|&&t| offers_a_summary(t)).count(); - for &q in &groups[g.0 as usize].queries { - most_approximate_inputs[q] += summaries; - } - } - (0..groups.len()) - .map(|g| match targets_of.get(&MemoGroupId(g as u32)) { - None => vec![own(g)], - Some(targets) => target_candidates( - per_computation, - groups, - &logical_nodes, - MemoGroupId(g as u32), - targets, - groups[g] - .queries - .iter() - .map(|&q| most_approximate_inputs[q]) - .max() - .unwrap_or(1) - .max(1), - ) - .unwrap_or_else(|| vec![own(g)]), - }) - .collect() -} - -/// The candidates of target aggregate group `g`, whose measures' targets -/// are `targets`: one per combination of their Pass 1 candidates (the last -/// target varying fastest), each the aggregate composed alone with that -/// choice. `None` when none composes. -fn target_candidates( - per_computation: &PerComputationCandidates, - groups: &[MemoGroup], - logical_nodes: &Rc<[Rc]>, - g: MemoGroupId, - targets: &[usize], - max_accuracy_budget_parts: usize, -) -> Option> { - let aggregate = Rc::clone(&groups[g.0 as usize].logical_node); - // The groups a candidate's private nodes may bottom out at, by key. - let mut below = HashSet::new(); - for &child in &groups[g.0 as usize].child_groups { - reach(groups, child, &mut below); - } - let mut structural_ids = NodeToStructuralIdCache::default(); - let mut descendant: HashMap = HashMap::new(); - let mut ordered: Vec = below.into_iter().collect(); - ordered.sort_unstable(); - for d in ordered { - descendant - .entry(structural_ids.structural_id(&groups[d.0 as usize].logical_node)) - .or_insert(d); - } - let own_key = structural_ids.structural_id(&groups[g.0 as usize].logical_node); - let mut candidates = Vec::new(); - // Each candidate's identity in the memo: its node's structural id, its - // input groups, the groups it fuses and its accuracy budget parts. A candidate - // equal to one already listed is not added again. - let mut known = HashSet::new(); - let mut choice = vec![0; per_computation.targets.len()]; - loop { - let mut implements = vec![g]; - for &t in targets { - if let Some((_, Some(absorbs))) = per_computation.targets[t].computation_for(choice[t]) - { - for &a in absorbs { - let absorbed = per_computation.targets[a].aggregate_group; - if !implements.contains(&absorbed) { - implements.push(absorbed); - } - } - } - } - for accuracy_budget_parts in 1..=max_accuracy_budget_parts { - let Ok(composed) = compose_alone( - per_computation, - logical_nodes, - &aggregate, - targets, - &choice, - accuracy_budget_parts, - ) else { - continue; - }; - let Some(QueryRoot::Operator(root)) = composed.into_iter().next() else { - continue; - }; - if accuracy_budget_parts > 1 && !builds_a_sketch(&root) { - break; - } - let (node, children) = if structural_ids.structural_id(&root) == own_key { - ( - Rc::clone(&groups[g.0 as usize].logical_node), - groups[g.0 as usize].child_groups.clone(), - ) - } else { - let mut children = Vec::new(); - let node = over_groups( - &root, - groups, - &mut structural_ids, - &descendant, - &mut children, - &mut HashMap::new(), - ); - // A parent reads the candidate in the group's place, so it - // outputs the group's schema when it has the same columns, - // with the group's metadata (composition may derive a closed - // label set where the frontend's node leaves it open). A - // query's top-k root may output other columns (the ranked - // value beside the key): it keeps its own. - let group_node = &groups[g.0 as usize].logical_node; - let same_columns = node.schema.fields == group_node.schema.fields - && node.schema.time_index == group_node.schema.time_index; - let node = if !same_columns || node.schema == group_node.schema { - node - } else { - Rc::new(OperatorNode { - schema: group_node.schema.clone(), - ..(*node).clone() - }) - }; - (node, children) - }; - if !known.insert(( - structural_ids.structural_id(&node), - children.clone(), - implements[1..].to_vec(), - accuracy_budget_parts, - )) { - continue; - } - candidates.push(CandidateForMemoGroup { - fused_groups: implements[1..].to_vec(), - root_node: node, - input_groups: children, - origin: OriginOfCandidateForMemoGroup::Pass1, - accuracy_budget_parts, - }); - } - // Mixed-radix over this group's targets only. - let mut advanced = false; - for &t in targets.iter().rev() { - choice[t] += 1; - if choice[t] < per_computation.targets[t].choice_count() { - advanced = true; - break; - } - choice[t] = 0; - } - if !advanced { - break; - } - } - (!candidates.is_empty()).then_some(candidates) -} - -/// `aggregate` composed alone with `choice[t]` for each of its measures' -/// `targets`, every other target left exact: the inventory holds only -/// `targets`, and the targets a chosen fused computation absorbs (not -/// computed then). -fn compose_alone( - per_computation: &PerComputationCandidates, - logical_nodes: &Rc<[Rc]>, - aggregate: &Rc, - targets: &[usize], - choice: &[usize], - accuracy_budget_parts: usize, -) -> Result, super::per_computation_candidates::PerComputationCandidateError> { - let mut kept: Vec = targets.to_vec(); - for &t in targets { - if let Some((_, Some(absorbs))) = per_computation.targets[t].computation_for(choice[t]) { - for &a in absorbs { - if !kept.contains(&a) { - kept.push(a); - } - } - } - } - let position = |t: usize| kept.iter().position(|&k| k == t); - let alone = PerComputationCandidates { - roots: vec![QueryRoot::Operator(Rc::clone(aggregate))], - logical_nodes: Rc::clone(logical_nodes), - targets: kept - .iter() - .map(|&t| { - let mut target = per_computation.targets[t].clone(); - if let Some(fused) = &mut target.fused_computation { - // Absorbed targets not kept are never absorbed here: the - // fusion is kept only when chosen, with all of them. - match fused - .absorbed_targets - .iter() - .map(|&a| position(a)) - .collect() - { - Some(absorbs) => fused.absorbed_targets = absorbs, - None => target.fused_computation = None, - } - } - target - }) - .collect(), - pruned_ways_to_compute: Vec::new(), - }; - let alone_choice: Vec = kept - .iter() - .map(|&t| if targets.contains(&t) { choice[t] } else { 0 }) - .collect(); - compose_logical_candidate_with_accuracy_budget_parts( - &alone, - &alone_choice, - accuracy_budget_parts, - ) -} - -/// `node` with each first node on a path down from its children that -/// equals a group in `descendant` replaced by that group's node (the same -/// `Rc`), and those groups pushed to `children` in the order met: the -/// candidate's private nodes over its child groups. -fn over_groups( - node: &Rc, - groups: &[MemoGroup], - structural_ids: &mut NodeToStructuralIdCache, - descendant: &HashMap, - children: &mut Vec, - rebuilt: &mut HashMap<*const OperatorNode, Rc>, -) -> Rc { - let mut replaced: HashMap<*const OperatorNode, Rc> = HashMap::new(); - for child in node.children() { - let key = Rc::as_ptr(child); - if replaced.contains_key(&key) { - continue; - } - let new = match descendant.get(&structural_ids.structural_id(child)) { - Some(&group) => { - if !children.contains(&group) { - children.push(group); - } - Rc::clone(&groups[group.0 as usize].logical_node) - } - None => match rebuilt.get(&key) { - Some(done) => Rc::clone(done), - None => { - let done = - over_groups(child, groups, structural_ids, descendant, children, rebuilt); - rebuilt.insert(key, Rc::clone(&done)); - done - } - }, - }; - replaced.insert(key, new); - } - if node - .children() - .into_iter() - .all(|child| Rc::ptr_eq(child, &replaced[&Rc::as_ptr(child)])) - { - return Rc::clone(node); - } - Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| Rc::clone(&replaced[&Rc::as_ptr(child)])), - ..(**node).clone() - }) -} - -/// Whether `node`'s private nodes build a sketch, which its accuracy budget parts size. -fn builds_a_sketch(node: &Rc) -> bool { - OperatorNode::reachable(node).iter().any(|n| { - matches!( - n.asap(), - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - } | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(..), - .. - } - ) - ) - }) -} - -/// Interned structural identities: the operator over its children's identities, -/// without the schema, guarantee, timing and materialization composition -/// derives. Memoized per node (the cache keeps each node alive, so its -/// address is not reused). -#[derive(Default)] -pub(crate) struct NodeToStructuralIdCache { - id_of_node: HashMap<*const OperatorNode, (Rc, u64)>, - id_of_encoded_operator: HashMap, u64>, -} - -impl NodeToStructuralIdCache { - pub(crate) fn structural_id(&mut self, node: &Rc) -> u64 { - if let Some((_, key)) = self.id_of_node.get(&Rc::as_ptr(node)) { - return *key; - } - let children: HashMap<*const OperatorNode, u64> = node - .children() - .into_iter() - .map(|child| (Rc::as_ptr(child), self.structural_id(child))) - .collect(); - let operator = node.operator.map_children(|c| children[&Rc::as_ptr(c)]); - // MessagePack preserves non-finite floats and signed zero. HashMap - // compares the complete byte key, so a hash collision cannot merge - // different computations. Cloned/recomposed NaN literals retain identity. - let encoded = - rmp_serde::to_vec(&operator).expect("an IR operator over child IDs must serialize"); - let next = self.id_of_encoded_operator.len() as u64; - let key = *self.id_of_encoded_operator.entry(encoded).or_insert(next); - self.id_of_node - .insert(Rc::as_ptr(node), (Rc::clone(node), key)); - key - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::stage0::generate_dag_of_logical_memo_groups; - - #[test] - fn structural_keys_preserve_nonfinite_and_signed_zero_literals() { - let input = crate::test_support::lower_promql( - "m", - asap_types::ir::properties::AccuracyTarget::exact(), - ); - let literal = |value| { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - asap_types::ir::NonASAPOp::Project { - cols: vec![asap_types::ir::ProjectItem { - alias: Some("value".into()), - expr: asap_types::ir::ScalarExpr::literal_f64(value), - }], - qualifier: None, - child: Rc::clone(&input), - }, - )) - .unwrap() - }; - let mut structural_ids = NodeToStructuralIdCache::default(); - for (left, right) in [ - (f64::NAN, f64::INFINITY), - (f64::INFINITY, f64::NEG_INFINITY), - (0.0, -0.0), - ] { - assert_ne!( - structural_ids.structural_id(&literal(left)), - structural_ids.structural_id(&literal(right)) - ); - } - assert_eq!( - structural_ids.structural_id(&literal(1.0)), - structural_ids.structural_id(&literal(1.0)) - ); - let node = literal(f64::NAN); - assert_eq!( - structural_ids.structural_id(&node), - structural_ids.structural_id(&literal(f64::NAN)) - ); - } - - fn populate_pass1( - initial: DagOfLogicalMemoGroups, - inventory: &PerComputationCandidates, - ) -> Memo { - try_populate_pass1(initial, inventory).unwrap() - } - - fn memo_of(inventory: &PerComputationCandidates) -> Memo { - let roots: Vec<_> = inventory.roots.clone(); - populate_pass1(generate_dag_of_logical_memo_groups(&roots), inventory) - } - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use crate::test_support::lower_promql; - use asap_types::ir::properties::AccuracyTarget; - - use asap_types::ir::ASAPOp; - - /// The cross-stage API rejects mismatched root identities without panicking. - #[test] - fn population_rejects_an_inventory_from_other_roots() { - let roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.95, m[5m])", - AccuracyTarget::max_error(0.01), - ))]; - let initial = generate_dag_of_logical_memo_groups(&roots); - // An inventory is tied to its roots by structure: the same query - // text lowered again is the same DAG, so the other roots must differ. - let other_roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.5, n[1m])", - AccuracyTarget::max_error(0.01), - ))]; - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&other_roots)) - .unwrap(); - assert!(matches!( - try_populate_pass1(initial, &inventory), - Err(CandidatePopulationError::RootDependenciesChanged) - )); - } - - /// Invalid target and fusion indices fail at the stage boundary, not in composition. - #[test] - fn population_checks_target_and_absorption_indices() { - let roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.95, m[5m])", - AccuracyTarget::max_error(0.01), - ))]; - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let mut invalid_measure = inventory.clone(); - invalid_measure.targets[0].measure_index = usize::MAX; - assert!(matches!( - try_populate_pass1(initial.clone(), &invalid_measure), - Err(CandidatePopulationError::InvalidTarget { target: 0 }) - )); - let mut invalid_absorption = inventory; - let missing = invalid_absorption.targets.len(); - invalid_absorption.targets[0].fused_computation = - Some(asap_types::stages::FusedComputation { - absorbed_targets: vec![missing], - ways_to_compute: Vec::new(), - }); - assert!(matches!(try_populate_pass1(initial, &invalid_absorption), - Err(CandidatePopulationError::InvalidAbsorption { target: 0, absorbed }) if absorbed == missing)); - } - - /// Capability pruning must change implementations, not logical identities. - #[test] - fn initial_groups_survive_pass1_expansion_and_pruning() { - let roots: Vec<_> = [ - "quantile_over_time(0.95, m[5m])", - "quantile_over_time(0.99, m[5m])", - ] - .iter() - .map(|text| QueryRoot::Operator(lower_promql(text, AccuracyTarget::max_error(0.01)))) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let original_groups = initial.groups.clone(); - let original_roots = initial.query_roots.clone(); - let inventory = generate_per_computation_candidates(&initial).unwrap(); - let expanded = populate_pass1(initial.clone(), &inventory); - let restricted_inventory = inventory.without_unsupported_summaries(Some(&[])); - let restricted = populate_pass1(initial, &restricted_inventory); - for memo in [&expanded, &restricted] { - assert_eq!(memo.groups.len(), original_groups.len()); - for (before, after) in original_groups.iter().zip(&memo.groups) { - assert!(Rc::ptr_eq(&before.logical_node, &after.logical_node)); - assert_eq!(before.child_groups, after.child_groups); - assert_eq!(before.parents, after.parents); - assert_eq!(before.queries, after.queries); - } - assert_eq!(memo.query_roots, original_roots); - assert_ne!( - group_of_kind(memo, 0, "Aggregate"), - group_of_kind(memo, 1, "Aggregate") - ); - assert!(memo.candidates.iter().all(|c| !c.is_empty())); - } - assert!( - expanded.candidates.iter().map(Vec::len).sum::() - > restricted.candidates.iter().map(Vec::len).sum::() - ); - } - - /// The memo of PromQL `queries`, each with accuracy target `target`. - fn memo(queries: &[&str], target: AccuracyTarget) -> Memo { - let roots: Vec<_> = queries - .iter() - .map(|q| QueryRoot::Operator(lower_promql(q, target.clone()))) - .collect(); - memo_of( - &generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(), - ) - } - - /// The group of the first node of `kind` reached from query `q`'s root, - /// walking down the groups' first children. - fn group_of_kind(memo: &Memo, q: usize, kind: &str) -> usize { - let mut g = memo.query_roots[q].as_operator().unwrap().0 as usize; - while memo.groups[g].logical_node.operator.kind_name() != kind { - g = memo.groups[g].child_groups[0].0 as usize; - } - g - } - - /// The summary algorithms a candidate's private nodes build. - fn algorithms(candidate: &CandidateForMemoGroup) -> Vec { - OperatorNode::reachable(&candidate.root_node) - .iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => Some(format!("{family:?}")), - _ => None, - }) - .collect() - } - - /// The per-job rate sum both queries compute is one group, reached by - /// both, the root of the first and read by the second's arithmetic. - #[test] - fn identical_sub_expressions_of_two_queries_are_one_group() { - let memo = memo( - &[ - "sum by (job) (rate(m[5m]))", - "sum by (job) (rate(m[5m])) * 2", - ], - AccuracyTarget::exact(), - ); - let sum = group_of_kind(&memo, 0, "Aggregate"); - assert_eq!(group_of_kind(&memo, 1, "Aggregate"), sum); - let group = &memo.groups[sum]; - assert_eq!(group.queries, vec![0, 1]); - assert_eq!( - asap_types::stages::queries_rooted_at(&memo.query_roots, MemoGroupId(sum as u32)), - vec![0] - ); - let product = memo.query_roots[1].as_operator().unwrap().0 as usize; - assert_eq!(group.parents, vec![MemoGroupId(product as u32)]); - assert_eq!( - memo.groups[product].child_groups, - vec![MemoGroupId(sum as u32)] - ); - // Children come before parents. - for (g, group) in memo.groups.iter().enumerate() { - assert!(group.child_groups.iter().all(|c| (c.0 as usize) < g)); - } - } - - /// A quantile's aggregate group has an exact candidate (its own node over - /// its input's group) and a KLL and a DDSketch candidate over the same - /// input group. - #[test] - fn a_quantile_target_group_has_exact_kll_and_ddsketch_candidates() { - let memo = memo( - &["quantile_over_time(0.9, m[5m])"], - AccuracyTarget::max_error(0.01), - ); - let quantile = group_of_kind(&memo, 0, "Aggregate"); - let input = memo.groups[quantile].child_groups.clone(); - let candidates = &memo.candidates[quantile]; - assert_eq!(candidates.len(), 3); - assert!(Rc::ptr_eq( - &candidates[0].root_node, - &memo.groups[quantile].logical_node - )); - assert!(algorithms(&candidates[0]).is_empty()); - assert!(algorithms(&candidates[1])[0].contains("Kll")); - assert!(algorithms(&candidates[2])[0].contains("DDSketch")); - for candidate in candidates { - assert!(candidate.fused_groups.is_empty()); - assert_eq!(candidate.input_groups, input); - assert_eq!(candidate.origin, OriginOfCandidateForMemoGroup::Pass1); - } - } - - /// The whole-expression top-k heap over the per-job sums implements both - /// aggregates' groups, and reads the sums' input group; the exact top-k - /// reads the sums' group. - #[test] - fn a_fused_candidate_lists_both_groups_it_implements() { - let memo = memo( - &["topk by (job) (10, sum_over_time(m[1m]))"], - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - let topk = group_of_kind(&memo, 0, "Aggregate"); - let sums = memo.groups[topk].child_groups[0]; - let sums_input = memo.groups[sums.0 as usize].child_groups.clone(); - let candidates = &memo.candidates[topk]; - assert_eq!(candidates[0].input_groups, vec![sums]); - let fused: Vec<_> = candidates - .iter() - .filter(|c| !c.fused_groups.is_empty()) - .collect(); - assert!(!fused.is_empty()); - for candidate in fused { - assert_eq!(candidate.fused_groups, vec![sums]); - assert_eq!(candidate.input_groups, sums_input); - assert!(algorithms(candidate)[0].contains("Heap")); - } - } - - /// Every candidate reads its child groups through their nodes (the same - /// `Rc`), and outputs its group's schema, so a parent can read it in - /// the group's place. - #[test] - fn candidates_read_child_group_nodes_and_output_the_group_schema() { - for (queries, target) in [ - ( - vec!["quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])"], - AccuracyTarget::max_error(0.01), - ), - ( - vec!["topk by (job) (10, sum_over_time(m[1m]))"], - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ), - (vec!["sum by (job) (rate(m[5m]))"], AccuracyTarget::exact()), - ] { - let memo = memo(&queries, target); - for (g, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates { - let reached: HashSet<*const OperatorNode> = - OperatorNode::reachable(&candidate.root_node) - .iter() - .map(Rc::as_ptr) - .collect(); - for child in &candidate.input_groups { - let node = &memo.groups[child.0 as usize].logical_node; - assert!(reached.contains(&Rc::as_ptr(node)), "{queries:?} g{g}"); - } - assert_eq!( - candidate.root_node.schema, - memo.groups[g].logical_node.schema - ); - } - } - } - } - - /// The sketch summaries of the first query of `queries` composed with - /// every target at its `algorithm` summary, by [`super::per_computation_candidates::compose_logical_candidate`]: - /// as the exhaustive search sizes them. - fn composed_sketches( - queries: &[&str], - target: &AccuracyTarget, - algorithm: &str, - ) -> Vec { - let roots: Vec<_> = queries - .iter() - .map(|q| QueryRoot::Operator(lower_promql(q, target.clone()))) - .collect(); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(|c| { - matches!(c, HowToComputeAggregateMeasure::Summary(a) if format!("{a:?}") == algorithm) - }) - .unwrap() - }) - .collect(); - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - unreachable!("an operator root") - }; - OperatorNode::reachable(root) - .iter() - .filter_map(|n| match n.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => Some(format!("{family:?}")), - _ => None, - }) - .collect() - } - - /// The sketch candidates of `memo`'s group `g`: each one's summary and - /// accuracy budget parts. - fn sketch_shares(memo: &Memo, g: usize) -> Vec<(String, usize)> { - memo.candidates[g] - .iter() - .filter(|c| !algorithms(c).is_empty()) - .map(|c| (algorithms(c)[0].clone(), c.accuracy_budget_parts)) - .collect() - } - - /// `q0.9 / q0.5` beside a plain `q0.9`: each of the ratio's quantile - /// groups offers every sketch for the whole budget and for half of it, - /// since its query has two inputs that may be approximate. The plain - /// query's quantile is the numerator's group: equal per-series quantiles - /// return the same rows, so they are one group with both readers. A ratio of quantiles carries no - /// guarantee, so composition does not split its budget: the - /// whole-budget candidate is the exhaustive search's sizing, and the - /// half-budget one, larger, is never the cheaper valid one. - #[test] - fn a_ratio_operand_has_a_candidate_per_accuracy_budget_parts() { - let queries = [ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])", - "quantile_over_time(0.9, m[5m])", - ]; - let target = AccuracyTarget::max_error(0.01); - let memo = memo(&queries, target.clone()); - let ratio = memo.query_roots[0].as_operator().unwrap().0 as usize; - let operand = memo.groups[ratio].child_groups[0].0 as usize; - for operand in &memo.groups[ratio].child_groups { - let shares: Vec = sketch_shares(&memo, operand.0 as usize) - .into_iter() - .map(|(_, share)| share) - .collect(); - assert_eq!(shares, vec![1, 2, 1, 2]); - } - let plain = memo.query_roots[1].as_operator().unwrap().0 as usize; - assert_eq!(plain, operand); - let whole_kll = &sketch_shares(&memo, operand)[0]; - assert_eq!(whole_kll.1, 1); - assert_eq!( - composed_sketches(&queries[..1], &target, "Kll"), - vec![whole_kll.0.clone(); 2] - ); - } - - /// A difference of two Count-Min counts splits its budget: each - /// operand's half-budget candidate is the Count-Min composition sizes - /// for the difference with both counts sketched. - #[test] - fn a_split_budget_matches_its_half_budget_candidate() { - let queries = ["count by (job) (a) - count by (job) (b)"]; - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let memo = memo(&queries, target.clone()); - let difference = memo.query_roots[0].as_operator().unwrap().0 as usize; - let operand = memo.groups[difference].child_groups[0].0 as usize; - let half = sketch_shares(&memo, operand) - .into_iter() - .find(|(family, share)| family.contains("Cms") && *share == 2) - .unwrap(); - assert_eq!(composed_sketches(&queries, &target, "Cms"), vec![half.0; 2]); - } - - /// A group appended after its readers (as a transformation appends a - /// ranking group) is ordered before every group whose candidate reads - /// it. - #[test] - fn topological_order_puts_appended_groups_before_their_readers() { - let mut memo = memo( - &["quantile_over_time(0.9, m[5m])"], - AccuracyTarget::max_error(0.01), - ); - assert_eq!( - memo.topological_order().unwrap(), - (0..memo.groups.len() as u32) - .map(MemoGroupId) - .collect::>() - ); - let quantile = group_of_kind(&memo, 0, "Aggregate"); - let appended = MemoGroupId(memo.groups.len() as u32); - memo.groups.push(memo.groups[1].clone()); - memo.candidates.push(memo.candidates[1].clone()); - let mut reader = memo.candidates[quantile][0].clone(); - reader.input_groups = vec![appended]; - memo.candidates[quantile].push(reader); - let order = memo.topological_order().unwrap(); - let position = |g: MemoGroupId| order.iter().position(|&x| x == g).unwrap(); - assert!(position(appended) < position(MemoGroupId(quantile as u32))); - // A cycle has no order. - memo.candidates[1][0] - .input_groups - .push(MemoGroupId(quantile as u32)); - assert_eq!(memo.topological_order(), None); - } - - /// The workload generator of `joint_search_equivalence`: a seeded - /// xorshift generator. - struct Rng(u64); - - impl Rng { - fn below(&mut self, n: usize) -> usize { - self.0 ^= self.0 << 13; - self.0 ^= self.0 >> 7; - self.0 ^= self.0 << 17; - (self.0 % n as u64) as usize - } - - fn pick<'a, T>(&mut self, items: &'a [T]) -> &'a T { - &items[self.below(items.len())] - } - } - - /// One random PromQL query of `joint_search_equivalence`. - fn random_query(rng: &mut Rng, gauges: &[&str]) -> String { - let m = rng.pick(gauges); - let group = rng.pick(&["job", "job, instance", "instance"]); - let window = rng.pick(&["1m", "5m", "10m"]); - let phi = rng.pick(&["0.5", "0.9", "0.99"]); - let k = rng.pick(&["3", "5"]); - match rng.below(10) { - 0 => format!("sum by ({group}) (sum_over_time({m}[{window}]))"), - 1 => format!("avg by ({group}) (quantile_over_time({phi}, {m}[{window}]))"), - 2 => format!("quantile_over_time({phi}, {m}[{window}])"), - 3 => format!("topk by ({group}) ({k}, sum_over_time({m}[{window}]))"), - 4 => format!("sum by ({group}) (rate(c_total[{window}]))"), - 5 => format!("max by ({group}) (max_over_time({m}[{window}]))"), - 6 => format!("count_over_time({m}[{window}])"), - 7 => format!("sum by ({group}) (increase(c_total[{window}]))"), - 8 => format!("stddev_over_time({m}[{window}])"), - _ => format!("topk({k}, sum by ({group}) (rate(c_total[{window}])))"), - } - } - - /// The roots of `joint_search_equivalence`'s random workload `seed` - /// over `gauges`, lowered as the planner lowers PromQL (with series - /// identity). - fn random_roots(seed: u64, gauges: &[&str]) -> Vec { - use asap_types::ir::properties::AccuracyMetric; - use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, - PrometheusMetricType, Query, QueryLanguage, QueryRequirements, QueryWorkload, - RepeatedDemand, RepeatingEntry, RepetitionInterval, SourceFacts, - }; - let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); - let queries = 2 + rng.below(3); - let entries = (0..queries) - .map(|_| RepeatingEntry { - query: Query(random_query(&mut rng, gauges)), - demand: RepeatedDemand::FixedInterval(RepetitionInterval( - *rng.pick(&[60_000u32, 300_000]), - )), - requirements: QueryRequirements { - accuracy: rng - .pick(&[ - AccuracyTarget::exact(), - AccuracyTarget::max_error(0.05), - AccuracyTarget::max_error(0.01).failure_probability(0.01), - AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Rank), - ]) - .clone(), - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - }) - .collect(); - let source = |metric_type| SourceFacts { - series_count: Some(1_000), - metric_type: Some(metric_type), - sample_interval: Some(DurationMs(15_000)), - ..SourceFacts::default() - }; - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(entries), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 2_000.0 / 15.0, - }, - sources: gauges - .iter() - .map(|g| (g.to_string(), source(PrometheusMetricType::Gauge))) - .chain([("c_total".to_string(), source(PrometheusMetricType::Counter))]) - .collect(), - }), - }; - asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryRoot::Operator( - asap_types::ir::schema::schema_support::with_promql_series_identity(&node) - .unwrap(), - ), - scalar => scalar, - }) - .collect() - } - - /// Whether a composed sub-DAG is some combination of the memo's - /// per-group candidates: a candidate's private nodes equal the composed - /// nodes operator by operator (schemas and guarantees aside, which - /// composition derives), and each child group's node is matched by a - /// candidate of that group in turn. - struct Combination<'a> { - memo: &'a Memo, - matched: HashMap<(usize, *const OperatorNode), bool>, - own: HashMap<*const OperatorNode, (Rc, serde_json::Value)>, - } - - impl Combination<'_> { - fn own_fields(&mut self, node: &Rc) -> serde_json::Value { - self.own - .entry(Rc::as_ptr(node)) - .or_insert_with(|| { - let operator = node.operator.map_children(|_| ()); - (Rc::clone(node), serde_json::to_value(&operator).unwrap()) - }) - .1 - .clone() - } - - fn group(&mut self, g: usize, composed: &Rc) -> bool { - let key = (g, Rc::as_ptr(composed)); - if let Some(&done) = self.matched.get(&key) { - return done; - } - self.matched.insert(key, false); - let memo = self.memo; - let found = memo.candidates[g] - .iter() - .any(|c| self.private(&c.root_node, composed, &c.input_groups)); - self.matched.insert(key, found); - found - } - - fn private( - &mut self, - node: &Rc, - composed: &Rc, - children: &[MemoGroupId], - ) -> bool { - let memo = self.memo; - if let Some(child) = children - .iter() - .find(|c| Rc::ptr_eq(&memo.groups[c.0 as usize].logical_node, node)) - { - return self.group(child.0 as usize, composed); - } - let (ours, theirs) = (node.children(), composed.children()); - ours.len() == theirs.len() - && self.own_fields(node) == self.own_fields(composed) - && ours - .iter() - .zip(&theirs) - .all(|(a, b)| self.private(a, b, children)) - } - } - - /// For the random workloads of `joint_search_equivalence` (over one - /// gauge and over two), every whole-workload Pass 1 choice composes, - /// for each query, a plan that is a combination of the memo's per-group - /// candidates: the memo misses no Pass 1 candidate. Every target's - /// group has the target's own and fused candidates, also a target a - /// fused computation absorbs; every candidate of a group a parent reads - /// outputs the group's schema. - #[test] - fn every_composed_pass1_candidate_is_a_combination_of_group_candidates() { - let mut checked = 0; - let mut missing = Vec::new(); - let mut other_schema = Vec::new(); - for gauges in [&["m"][..], &["m", "n"]] { - for seed in 1..=60u64 { - let roots = random_roots(seed, gauges); - let inventory = generate_per_computation_candidates( - &generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - if inventory.combinations() > 400 { - continue; - } - let memo = memo_of(&inventory); - // Each target's own and fused candidates, absorbed or not. - let group_of = |t: usize| { - memo.groups - .iter() - .position(|group| { - *group.logical_node == **inventory.aggregate(&inventory.targets[t]) - }) - .unwrap() - }; - for (t, target) in inventory.targets.iter().enumerate() { - let g = group_of(t); - if inventory.targets.iter().enumerate().any(|(u, other)| { - u != t && group_of(u) == g && other.measure_index != target.measure_index - }) { - continue; - } - let whole_budget = |fused: bool| { - memo.candidates[g] - .iter() - .filter(|c| { - c.accuracy_budget_parts == 1 && !c.fused_groups.is_empty() == fused - }) - .count() - }; - let fused = target - .fused_computation - .as_ref() - .map_or(0, |f| f.ways_to_compute.len()); - if whole_budget(false) != target.ways_to_compute.len() - || whole_budget(true) != fused - { - missing.push(format!("{gauges:?} seed {seed} target {t} candidates")); - } - } - for (g, group) in memo.groups.iter().enumerate() { - for candidate in &memo.candidates[g] { - if !group.parents.is_empty() - && candidate.root_node.schema != group.logical_node.schema - { - other_schema.push(format!("{gauges:?} seed {seed} g{g}")); - } - } - } - let root_group = |q: usize| memo.query_roots[q].as_operator().unwrap().0 as usize; - let mut combination = Combination { - memo: &memo, - matched: HashMap::new(), - own: HashMap::new(), - }; - let mut choice = vec![0; inventory.targets.len()]; - loop { - if !inventory.absorbed_choice_varies(&choice) { - if let Ok(composed) = compose_logical_candidate(&inventory, &choice) { - for (q, root) in composed.iter().enumerate() { - let QueryRoot::Operator(node) = root else { - continue; - }; - checked += 1; - if !combination.group(root_group(q), node) { - missing.push(format!("{gauges:?} seed {seed} q{q} {choice:?}")); - } - } - } - } - let mut advanced = false; - for t in (0..choice.len()).rev() { - choice[t] += 1; - if choice[t] < inventory.targets[t].choice_count() { - advanced = true; - break; - } - choice[t] = 0; - } - if !advanced { - break; - } - } - } - } - assert!(checked > 100, "{checked} plans checked"); - assert!( - missing.is_empty(), - "{} of {checked} missing: {missing:#?}", - missing.len() - ); - } - - /// Two SQL queries over one table, each lowered with its own scan, read - /// one scan group: equal scans are shared, a table's keyless rows - /// included. Their aggregates, grouped differently, stay two groups. - #[test] - fn equal_sql_table_scans_are_one_group() { - use crate::test_support::{agg, scan_from}; - use asap_types::ir::operator::{AggIntent, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - let table = || { - scan_from( - Source::Table { - table_ref: "t".into(), - }, - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("a", DataType::Utf8, true), - Field::plain("b", DataType::Utf8, true), - ], - 0, - vec![], - ), - ) - }; - let sum = || AggIntent::Sum { col: Some(1) }; - let roots = vec![ - QueryRoot::Operator(agg(vec![2], sum(), table())), - QueryRoot::Operator(agg(vec![3], sum(), table())), - ]; - let memo = memo_of( - &generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(), - ); - assert_eq!(memo.groups.len(), 3); - assert_eq!(memo.groups[0].logical_node.operator.kind_name(), "Scan"); - assert_eq!(memo.groups[0].queries, vec![0, 1]); - assert_eq!(memo.groups[0].parents, vec![MemoGroupId(1), MemoGroupId(2)]); - } -} diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs deleted file mode 100644 index ea2b891ca..000000000 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ /dev/null @@ -1,4922 +0,0 @@ -//! Stage 1 (logical ASAP-aware optimization), Pass 1: per-computation -//! candidate generation, as defined in -//! `docs/design_docs/proposals/planner-layering.md`. -//! -//! [`generate_per_computation_candidates`] walks the memo groups of the -//! workload's hash-consed logical DAG ([`DagOfLogicalMemoGroups`]) and -//! returns, for every eligible computation, its unranked candidates. A -//! computation ([`ComputationTarget`]) is one measure of an `Aggregate` -//! group whose intent has a realization besides plain exact execution, once -//! however many queries read the group; nested aggregates and operators -//! read by scalar roots are found too. -//! Pass 1 does not consider sharing across computations; that is Pass 2. -//! -//! - A candidate says how a target is computed: exact or a summary -//! ([`HowToComputeAggregateMeasure`]). Each target has exactly one exact option: -//! its exact accumulator states (`ExactAccumulator`; `[Sum, Count]` for -//! `avg`) when they can be built, otherwise plain `Exact`. Whether they are -//! materialized is decided in Stage 2. -//! - A computation that fuses a target with the targets beneath it is the -//! outer target's [`FusedComputation`], which absorbs the ones beneath and -//! has its own candidates: the whole-expression top-k over a per-item -//! `sum` or `count` (`Summary` of a heap family) and the nested sum, a -//! grouped `sum` over a `sum` (`ExactAccumulator([Sum])`). -//! - Summary choices here are unsized families. Attaching them to memo -//! groups creates sized candidates, one per split of the accuracy budget -//! (`CandidateForMemoGroup::accuracy_budget_parts`), each sized for ε, its -//! accuracy budget parts and δ over the values its readers' targets cover; Pass 2 -//! inherits sizes or takes the strictest shared reader's, and Stage 3 only -//! checks. -//! - Candidates are pruned only when provably unable to meet the accuracy -//! requirement or to run, or when the exact candidate provably dominates -//! them (a PromQL top-k's heap over its instant vector, one row per item: -//! [`PruneReason::TopKHeapOverOneRowPerItem`]); each pruned candidate -//! keeps its [`PruneReason`]. -//! - A measure filter or `HAVING` prunes nothing. The target records the -//! measure filter (applied as `SummaryAgg.filter`) and one [`HavingEntry`] -//! per consumer `Filter` directly above the aggregate: a key-only predicate -//! applied by every reader of the aggregate's group, across queries, may go -//! below the aggregate, anything else stays above the output (approximate -//! outputs then get guarantee `None`). So every candidate of a group -//! computes the group's own rows. -//! -//! The input must be a logical DAG: ASAP operators, execution timing or a -//! materialization are rejected. No cost model is used. The accuracy model -//! is used only to prune summaries no query reading them can use -//! ([`PerComputationCandidates::without_summaries_no_query_can_use`]); -//! ranking and discarding valid candidates belong to Stage 3. -//! -//! # Whole-workload candidates (logical ASAP-aware optimization, end of Pass 1) -//! -//! [`compose_logical_candidate`] is Algorithm 1's -//! `SUBSTITUTE(d, s_1 ↦ c_1, …)`: it picks one per-computation candidate per -//! target (`choice[i]` indexes `targets[i]`'s candidates, then its fused -//! computation's, [`ComputationTarget::computation_for`]) and builds one whole-workload -//! candidate DAG. An aggregate whose chosen candidates are all `Exact` is -//! kept. Otherwise each of its measures is substituted on its own over the -//! aggregate's input: -//! -//! - `ExactAccumulator` with one state: `SummaryAgg` with the exact family → -//! `FinalizeExactAccumulator`. -//! - `ExactAccumulator([Sum, Count])` of an `avg`: the finalized `sum` and -//! `count` accumulators divided: a PromQL `BinaryOp`, or for SQL rows a -//! join on the group keys (NULL keys match) and a SQL division, NULL where -//! the count is 0. -//! - `Summary`: `SummaryAgg` → `SummaryEstimate`. -//! - `Exact` (or a measure that is no target): a single-measure `Aggregate`. -//! -//! A fused computation's candidate reads the absorbed targets' input instead: -//! the whole-expression top-k is one heap `SummaryAgg` keyed by the ranked -//! item → `SummaryEstimate` (over a grouped sum of per-series sums or counts, -//! it reads the per-series target's input, the raw data samples); the nested sum is `SummaryAgg` with the exact -//! `Sum` family grouped by the outer keys → `FinalizeExactAccumulator`. -//! -//! The substituted output keeps the aggregate's column names. The measures -//! of a multi-measure aggregate are joined back on the group keys (NULL keys -//! match, as grouping treats them). A measure filter becomes -//! `SummaryAgg.filter`, so a group with no matching row stays. A -//! `BelowAggregate` `HAVING` becomes a `Filter` below the state, with its key -//! columns renumbered to the input's, and the consumer `Filter` it came from -//! is dropped; it is not a `SummaryAgg.filter`, which would keep the dropped -//! groups. An `AboveOutput` `HAVING` stays a `Filter` above the output. -//! -//! Summaries are sized here, each for its own measure's accuracy requirement. -//! When a query's value composes several approximate inputs (every sketch -//! summary below its root, when its guarantee is not exact), each is sized -//! for an equal share of that requirement: `ε/n` (a relative error -//! `1 − (1 + ε)^{−1/n}`) and `δ/n` for `n` inputs (#669b). Only the inputs -//! approximate in this candidate count. This composition sizes for ε and δ -//! per output value; attached to the memo groups, each candidate's -//! approximate inputs are sized for the values its readers' targets cover -//! (`δ / (n · V)`, -//! [`generate_per_computation_logical_memo_candidates`](crate::pass1::logical_memo_candidates::generate_per_computation_logical_memo_candidates)), -//! their final sizes. Pass 2 inherits them or takes the strictest shared -//! reader's, and Stage 3 only checks the composed accuracy. -//! -//! Every node of the candidate gets a theoretical guarantee, composed with -//! the built-in accuracy model through operators (#669): exact over exact -//! inputs; a `SummaryEstimate` the sized family's bound where that bound -//! answers the estimate (not for Top-K, where a heap sketch bounds item -//! frequencies, not membership; not for a Count-Min family over weights not -//! proven non-negative); aggregation and arithmetic over approximate values -//! by their composition rules, with statistics left symbolic for plan -//! selection to bind; and `None` where no rule applies (comparisons and -//! value predicates, rank errors). Plan selection's accuracy model decides -//! a candidate without one. -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; -use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::{DataType, Schema}; -use asap_types::ir::schema::{ - EntityIdentity, ExactKind, FieldDataType, GroupingStrategy, NonNegativeWeightProof, - SketchAlgorithm, SketchKind, StatisticKind, SummaryInputExpr, SummaryUpdate, WeightDomain, -}; -use asap_types::ir::{ - ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, - Predicate, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, -}; -use asap_types::physical::Materialization; -use thiserror::Error; - -use asap_accuracy::composition::parse_raw_data_samples_sign_statistic; -use asap_accuracy::estimators::hll; -use asap_accuracy::propagation::{ - approximate_inputs, bind_guarantees, raw_data_samples_of_metric, root_guarantee, - with_guarantees, -}; -use asap_accuracy::sizing::{accuracy_budget, sized, sizing_target, sketch_statistic}; -use asap_accuracy::AccuracyModel; -use asap_types::ir::properties::AccuracyGuarantee; -use asap_types::workload::bounds::{declared_non_negative, series_read, values_per_result}; -use asap_types::workload::DataWorkload; - -use crate::deployment_capabilities::{summary_supported, SupportedSummary}; -use crate::pass1::realization::summary_candidates; -use asap_types::stages::{ - ComputationTarget, DagOfLogicalMemoGroups, FusedComputation, HavingEntry, HavingPlacement, - HowToComputeAggregateMeasure, MemoGroupId, PruneReason, -}; - -/// Unranked candidates of one measure and the candidates pruned for it. -#[derive(Debug, Clone, PartialEq)] -pub(super) struct ComputationChoices { - pub candidates: Vec, - pub pruned: Vec<(HowToComputeAggregateMeasure, PruneReason)>, -} - -impl ComputationChoices { - /// Prune matching candidates; a pruned exact accumulator form is replaced by - /// plain `Exact`, so the target keeps one exact option. - fn prune( - &mut self, - reason: PruneReason, - applies: impl Fn(&HowToComputeAggregateMeasure) -> bool, - ) { - let pruned = prune_own(&mut self.candidates, applies); - self.pruned - .extend(pruned.into_iter().map(|choice| (choice, reason))); - } -} - -/// Removes from a measure's own `candidates` the ones `applies` matches -/// (never `Exact`) and returns them; a pruned exact accumulator form is -/// replaced by plain `Exact`, so the measure keeps one exact option. -fn prune_own( - candidates: &mut Vec, - applies: impl Fn(&HowToComputeAggregateMeasure) -> bool, -) -> Vec { - let (pruned, kept): (Vec<_>, _) = std::mem::take(candidates) - .into_iter() - .partition(|choice| *choice != HowToComputeAggregateMeasure::Exact && applies(choice)); - *candidates = kept; - if pruned - .iter() - .any(|choice| matches!(choice, HowToComputeAggregateMeasure::ExactAccumulator(_))) - { - candidates.insert(0, HowToComputeAggregateMeasure::Exact); - } - pruned -} - -/// A way to compute a target's measure that Pass 1 pruned, and why. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct PrunedWayToCompute { - /// The target, by index in [`PerComputationCandidates::targets`]. - pub target: usize, - /// Whether it was a way of the target's fused computation - /// ([`ComputationTarget::fused_computation`]) rather than its own. - pub of_fused_computation: bool, - pub way_to_compute: HowToComputeAggregateMeasure, - pub reason: PruneReason, -} - -/// Compact Pass 1 inventory over the memo groups it walked: one target per -/// measure of each aggregate group, however many queries read the group. -/// Pass 1's working form, not its output: -/// [`generate_per_computation_logical_memo_candidates`](crate::pass1::generate_per_computation_logical_memo_candidates) -/// builds, prunes and attaches it to the memo groups. Public only to tests -/// (feature `test-support`). -#[derive(Debug, Clone)] -pub struct PerComputationCandidates { - /// Each query's root over the memo groups' logical nodes, in workload - /// order: a group several queries read is one node. Whole-workload - /// candidates are these roots with chosen candidates substituted. - pub roots: Vec, - /// Each memo group's logical node, by group index. - pub(crate) logical_nodes: Rc<[Rc]>, - /// The targets; a whole-workload choice has one digit per target, - /// ranging over [`ComputationTarget::choice_count`]. - pub targets: Vec, - /// The ways to compute the targets' measures that Pass 1 pruned, in the - /// order it pruned them. - pub(crate) pruned_ways_to_compute: Vec, -} - -impl PerComputationCandidates { - /// Whether this inventory walked `dag`, or a DAG built the same way from - /// the same roots: the same groups (by structural id, in the same order) - /// and each query's root over the same groups. Its targets' group ids - /// then name `dag`'s groups. - pub(crate) fn walked(&self, dag: &DagOfLogicalMemoGroups) -> bool { - use super::memo_population::NodeToStructuralIdCache; - let mut structural_ids = NodeToStructuralIdCache::default(); - let group_of: HashMap<*const OperatorNode, MemoGroupId> = self - .logical_nodes - .iter() - .enumerate() - .map(|(g, node)| (Rc::as_ptr(node), MemoGroupId(g as u32))) - .collect(); - dag.groups.len() == self.logical_nodes.len() - && dag.query_roots.len() == self.roots.len() - && dag - .groups - .iter() - .zip(self.logical_nodes.iter()) - .all(|(group, node)| { - Rc::ptr_eq(&group.logical_node, node) - || structural_ids.structural_id(&group.logical_node) - == structural_ids.structural_id(node) - }) - && dag - .query_roots - .iter() - .zip(&self.roots) - .all(|(groups, root)| { - let nodes = match root { - QueryRoot::Operator(node) => vec![node], - QueryRoot::Scalar(expr) => expr.operator_refs(), - }; - matches!(groups, QueryRoot::Operator(_)) - == matches!(root, QueryRoot::Operator(_)) - && groups.operator_refs().len() == nodes.len() - && groups - .operator_refs() - .into_iter() - .zip(nodes) - .all(|(g, node)| group_of.get(&Rc::as_ptr(node)) == Some(g)) - }) - } - - /// The `Aggregate` node of `target`'s memo group. - pub fn aggregate(&self, target: &ComputationTarget) -> &Rc { - &self.logical_nodes[target.aggregate_group.0 as usize] - } - - /// The number of whole-workload candidates: the size of the Cartesian - /// product of the targets' candidates, where a target with a fused - /// computation and the targets beneath that it absorbs count as their - /// own combinations plus the fused computation's candidates (the - /// absorbed targets are not computed then), saturating at `usize::MAX`. - /// A target with `n` candidates and `f` fused ones counts `n · Π m + f`, - /// the product over the targets it absorbs that no other of them - /// absorbs, each counted the same way. It is exactly the number of - /// choices enumeration keeps ([`Self::absorbed_choice_varies`]). - #[cfg(any(test, feature = "test-support"))] - pub fn combinations(&self) -> usize { - let absorbed: HashSet = absorbed_targets(&self.targets).collect(); - (0..self.targets.len()) - .filter(|index| !absorbed.contains(index)) - .map(|index| self.combinations_of(index)) - .fold(1usize, usize::saturating_mul) - } - - /// [`Self::combinations`] of target `index` with the targets it absorbs. - #[cfg(any(test, feature = "test-support"))] - fn combinations_of(&self, index: usize) -> usize { - let target = &self.targets[index]; - let n = target.ways_to_compute.len(); - let Some(fused) = &target.fused_computation else { - return n; - }; - let absorbed_by_another = |a: usize| { - fused.absorbed_targets.iter().any(|&b| { - self.targets[b] - .fused_computation - .as_ref() - .is_some_and(|inner| inner.absorbed_targets.contains(&a)) - }) - }; - fused - .absorbed_targets - .iter() - .filter(|&&a| !absorbed_by_another(a)) - .map(|&a| self.combinations_of(a)) - .fold(n, usize::saturating_mul) - .saturating_add(fused.ways_to_compute.len()) - } - - /// Whether `choice` picks a fused computation and a candidate other - /// than the first for a target it absorbs: the same whole-workload - /// candidate as the one with the first, so enumeration skips it. - #[cfg(any(test, feature = "test-support"))] - pub fn absorbed_choice_varies(&self, choice: &[usize]) -> bool { - self.targets.iter().zip(choice).any(|(target, &index)| { - target - .computation_for(index) - .and_then(|(_, absorbed)| absorbed) - .is_some_and(|inner| inner.iter().any(|&i| choice[i] != 0)) - }) - } - - /// The ways to compute target `target`'s measure that Pass 1 pruned, - /// with their reasons, in pruning order: its own, or its fused - /// computation's when `of_fused_computation`. - #[cfg(any(test, feature = "test-support"))] - pub fn pruned_of_target( - &self, - target: usize, - of_fused_computation: bool, - ) -> Vec<(HowToComputeAggregateMeasure, PruneReason)> { - self.pruned_ways_to_compute - .iter() - .filter(|pruned| { - pruned.target == target && pruned.of_fused_computation == of_fused_computation - }) - .map(|pruned| (pruned.way_to_compute.clone(), pruned.reason)) - .collect() - } - - /// This inventory without the summary candidates the executor does not - /// support (`supported_summaries`, the deployment's - /// `DeploymentCapabilities::supported_summaries`; `None` supports - /// every one): a target's own are moved to its pruned candidates with - /// [`PruneReason::UnsupportedByDeployment`], and so are a fused - /// computation's (with all of them pruned, the fusion is not offered). - /// The readout is the statistic the measure's estimate reads (the outer measure's for a - /// fused computation). With every summary supported it is unchanged. - pub fn without_unsupported_summaries( - mut self, - supported_summaries: Option<&[SupportedSummary]>, - ) -> Self { - for (index, target) in self.targets.iter_mut().enumerate() { - let aggregate = &self.logical_nodes[target.aggregate_group.0 as usize]; - let Some(NonASAPOp::Aggregate { measures, .. }) = aggregate.non_asap() else { - continue; - }; - let Some(readout) = sketch_statistic(&measures[target.measure_index]).map(|s| s.kind()) - else { - continue; - }; - let unsupported = |candidate: &HowToComputeAggregateMeasure| { - matches!(candidate, HowToComputeAggregateMeasure::Summary(family) - if !summary_supported(supported_summaries, family, readout)) - }; - let pruned = |of_fused_computation| { - move |way_to_compute| PrunedWayToCompute { - target: index, - of_fused_computation, - way_to_compute, - reason: PruneReason::UnsupportedByDeployment, - } - }; - let own = prune_own(&mut target.ways_to_compute, unsupported); - self.pruned_ways_to_compute - .extend(own.into_iter().map(pruned(false))); - if let Some(fused) = &mut target.fused_computation { - let (removed, kept): (Vec<_>, _) = std::mem::take(&mut fused.ways_to_compute) - .into_iter() - .partition(unsupported); - fused.ways_to_compute = kept; - self.pruned_ways_to_compute - .extend(removed.into_iter().map(pruned(true))); - } - } - self - } - - /// This inventory without the summary candidates no query can use - /// (#1047, accuracy A1 of #1095): a summary of a target (its own, or - /// its fused computation's) is moved to its pruned candidates with - /// [`PruneReason::NoErrorBoundForQueryTarget`] when, for every query - /// whose root reaches the target's aggregate, - /// `output_can_meet_target` says no: the query's output, composed - /// with that summary and every other target exact, has no error bound - /// that can meet the query's target (`query_targets[q]`) at any size, - /// as `model` judges metrics. Every other target exact is the most - /// favorable combination: an approximate input never adds a guarantee. - /// A summary some query can use is kept, even when another query - /// reaching the group cannot; plan selection rejects it for that query. - /// - /// Pass 1 cannot see Pass 2's reuse of one group's summary by another - /// group (rollup, window rollup, finer-aggregate reuse, top-k reuse), so - /// a summary is also kept whenever another target, over another - /// aggregate reading one of the same sources, offers the same family - /// and some query reading that target can use it: Pass 2 may build that - /// target's state from this one, and the reused state composes through - /// that target's queries as its own would. This keeps more than those - /// rules' preconditions need. Exact candidates are never - /// pruned, and with every fused summary pruned the fusion is not - /// offered. A summary whose composition fails is kept. - /// - /// Pass 1's entry (`generate_per_computation_logical_memo_candidates`) - /// calls it only when the deployment's accuracy model and evidence - /// cannot accept an output without its guarantee. - pub fn without_summaries_no_query_can_use( - mut self, - query_targets: &[AccuracyTarget], - data: &DataWorkload, - model: &dyn AccuracyModel, - ) -> Self { - // The aggregates each query's root reaches. - let reached: Vec> = self - .roots - .iter() - .map(|root| { - root.operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .filter(|node| matches!(node.non_asap(), Some(NonASAPOp::Aggregate { .. }))) - .map(|node| Rc::as_ptr(&node)) - .collect() - }) - .collect(); - // Each target's sources, and every family it offers. - let sources: Vec> = self - .targets - .iter() - .map(|target| sources_read(self.aggregate(target))) - .collect(); - let families = |target: &ComputationTarget| -> Vec { - (0..target.choice_count()) - .filter_map(|choice| match target.computation_for(choice) { - Some((HowToComputeAggregateMeasure::Summary(family), _)) => Some(*family), - _ => None, - }) - .collect() - }; - let offered: Vec> = self.targets.iter().map(families).collect(); - // Per target, its summary choices no query reading its aggregate can - // use; a target no query reaches, or whose composition fails, keeps - // every choice. - let mut unusable_by_own_queries: Vec> = vec![Vec::new(); self.targets.len()]; - for (index, target) in self.targets.iter().enumerate() { - let aggregate = Rc::as_ptr(self.aggregate(target)); - let queries: Vec = (0..self.roots.len()) - .filter(|&q| reached[q].contains(&aggregate) && q < query_targets.len()) - .collect(); - if queries.is_empty() { - continue; - } - let roots: Vec = queries.iter().map(|&q| self.roots[q].clone()).collect(); - for choice in 0..target.choice_count() { - let Some((HowToComputeAggregateMeasure::Summary(_), _)) = - target.computation_for(choice) - else { - continue; - }; - let mut choices = vec![0; self.targets.len()]; - choices[index] = choice; - let Ok(composed) = - compose_with_accuracy_budget_parts(&self, &roots, &choices, Some(1)) - else { - continue; - }; - if !queries - .iter() - .zip(&composed) - .any(|(&q, root)| output_can_meet_target(root, &query_targets[q], data, model)) - { - unusable_by_own_queries[index].push(choice); - } - } - } - // The families each target's own queries can use. - let usable_families: Vec> = self - .targets - .iter() - .enumerate() - .map(|(index, target)| { - (0..target.choice_count()) - .filter(|choice| !unusable_by_own_queries[index].contains(choice)) - .filter_map(|choice| match target.computation_for(choice) { - Some((HowToComputeAggregateMeasure::Summary(family), _)) => Some(*family), - _ => None, - }) - .collect() - }) - .collect(); - // Whether Pass 2 may reuse target `index`'s `family` for another - // target's aggregate whose own queries can use it: a reused state - // composes through that target's queries as its own would. - let reusable = |index: usize, family: &SketchAlgorithm| { - let aggregate = self.targets[index].aggregate_group; - self.targets.iter().enumerate().any(|(other, target)| { - target.aggregate_group != aggregate - && offered[other].contains(family) - && usable_families[other].contains(family) - && sources[other] - .iter() - .any(|source| sources[index].contains(source)) - }) - }; - let mut unusable: Vec<(usize, usize)> = Vec::new(); - for (index, choices) in unusable_by_own_queries.iter().enumerate() { - for &choice in choices { - let Some((HowToComputeAggregateMeasure::Summary(family), _)) = - self.targets[index].computation_for(choice) - else { - continue; - }; - if !reusable(index, family) { - unusable.push((index, choice)); - } - } - } - for (index, target) in self.targets.iter_mut().enumerate() { - let own = target.ways_to_compute.len(); - let pruned_choices: Vec = unusable - .iter() - .filter(|(i, _)| *i == index) - .map(|(_, choice)| *choice) - .collect(); - if pruned_choices.is_empty() { - continue; - } - let own_pruned: Vec = pruned_choices - .iter() - .filter(|&&choice| choice < own) - .map(|&choice| target.ways_to_compute[choice].clone()) - .collect(); - let fused_pruned: Vec = pruned_choices - .iter() - .filter(|&&choice| choice >= own) - .filter_map(|&choice| { - let fused = target.fused_computation.as_ref()?; - fused.ways_to_compute.get(choice - own).cloned() - }) - .collect(); - let pruned = |of_fused_computation| { - move |way_to_compute| PrunedWayToCompute { - target: index, - of_fused_computation, - way_to_compute, - reason: PruneReason::NoErrorBoundForQueryTarget, - } - }; - let removed = prune_own(&mut target.ways_to_compute, |way| own_pruned.contains(way)); - self.pruned_ways_to_compute - .extend(removed.into_iter().map(pruned(false))); - if let Some(fused) = &mut target.fused_computation { - let (removed, kept): (Vec<_>, _) = std::mem::take(&mut fused.ways_to_compute) - .into_iter() - .partition(|way| fused_pruned.contains(way)); - fused.ways_to_compute = kept; - self.pruned_ways_to_compute - .extend(removed.into_iter().map(pruned(true))); - } - } - self - } -} - -/// The sources every scan below `node` reads, each once. -fn sources_read(node: &Rc) -> Vec { - let mut sources = Vec::new(); - for node in OperatorNode::reachable(node) { - if let Some(NonASAPOp::Scan { source, .. }) = node.non_asap() { - if !sources.contains(source) { - sources.push(source.clone()); - } - } - } - sources -} - -/// Whether a query's composed output `root` can meet `target` at some size -/// of its summaries: its guarantee is exact, or it has one in a metric -/// `model` says can meet `target` ([`AccuracyModel::metric_can_meet`]; an -/// exact target accepts only exact) whose error, and failure probability -/// when `target` sets one, are known once the statistics Pass 1 left -/// symbolic are bound and the guarantee is stated over the values the -/// target's scope covers, as plan selection states it. The sign of a -/// metric's samples is bound as plan selection binds it, proven only when -/// `data` declares the metric a counter or non-negative -/// ([`declared_non_negative`]). Classic HLL's failure probability is bound -/// from `data`'s declared distinct bound for its source at the largest -/// precision the contract covers: a bound outside the contract there (no -/// certificate, failure probability 1) is outside it at every size, so it -/// stays unbound. Every other statistic (row and series counts, stream -/// norms, an undeclared distinct bound, operand domains) is assumed -/// bindable, so no candidate is pruned for one plan selection may bind. -/// The number of values a whole-result target covers is `data`'s -/// ([`values_per_result`]): when it is not bound, a per-value guarantee -/// that may fail never meets a whole-result failure probability, whatever -/// the size (sizing splits δ over those values). -pub(crate) fn output_can_meet_target( - root: &QueryRoot, - target: &AccuracyTarget, - data: &DataWorkload, - model: &dyn AccuracyModel, -) -> bool { - let bind = |guarantee: &AccuracyGuarantee| { - guarantee.bind(&|statistic| { - if let Some(metric) = parse_raw_data_samples_sign_statistic(statistic) { - return declared_non_negative(data, metric).then_some(1.0); - } - if let Some((_, relative_error, source)) = - hll::parse_classic_failure_statistic(statistic) - { - // At the largest precision the contract covers: no - // certificate there, none at any size. - let declared = data - .sources - .get(source) - .and_then(|facts| facts.max_distinct_per_evaluation); - if let Some(bound) = declared { - let best = - hll::classic_failure_probability(hll::MAX_PRECISION, relative_error, bound); - return (best < 1.0).then_some(best); - } - } - Some(1.0) - }) - }; - let guarantee = match root_guarantee(root) { - Some(guarantee) => Some(bind(&guarantee)), - None => bind_guarantees(root, &bind), - }; - let Some(guarantee) = guarantee else { - return false; - }; - if guarantee.is_exact() { - return true; - } - let values = values_per_result(root, series_read(root, data), data); - let guarantee = guarantee.over_scope(target.accuracy_scope, values); - model.metric_can_meet(guarantee.spec.accuracy_metric, target) - && guarantee.spec.error.evaluate().is_some() - && (target.failure_probability.is_none() - || guarantee.spec.failure_probability.evaluate().is_some()) -} - -/// The indices of `targets` some target's fused computation absorbs. -#[cfg(any(test, feature = "test-support"))] -fn absorbed_targets(targets: &[ComputationTarget]) -> impl Iterator + '_ { - targets - .iter() - .filter_map(|target| target.fused_computation.as_ref()) - .flat_map(|fused| fused.absorbed_targets.iter().copied()) -} - -#[derive(Debug, Error)] -pub enum PerComputationCandidateError { - #[error(transparent)] - Structure(#[from] SchemaDerivationError), - #[error("logical candidate input already has assigned execution timing or materialization")] - AssignedPhysicalAnnotation, - #[error("logical candidate input already contains ASAP operators")] - ContainsAsap, - /// `root` (the query's index in the workload) and `measure` locate the target; - /// `computation_candidates_for_intent` alone leaves them `None`. - #[error( - "approximate accuracy of root {}, measure {} requires epsilon and delta in (0, 1)", - root.as_deref().unwrap_or("?"), - measure.map_or_else(|| "?".to_string(), |measure| measure.to_string()) - )] - InvalidAccuracy { - root: Option, - measure: Option, - }, - #[error("choice must name one listed candidate per target")] - InvalidChoice, - #[error("unsupported substitution in logical ASAP-aware optimization: {0}")] - Unsupported(&'static str), -} - -/// Enumerate exact, rewrite and summary choices of one measure in stable catalog -/// order, without ranking or empirical assessment. A summary is not a claim that -/// a deployment meets the measure's accuracy requirement. The one exact option -/// comes first: the exact accumulator form if the intent has one, else `Exact`. -pub(super) fn computation_candidates_for_intent( - intent: &AggIntent, -) -> Result { - let states = match intent { - AggIntent::Count { .. } => vec![ExactKind::Count], - AggIntent::Sum { .. } => vec![ExactKind::Sum], - AggIntent::Min { .. } => vec![ExactKind::Min], - AggIntent::Max { .. } => vec![ExactKind::Max], - AggIntent::Rate => vec![ExactKind::Rate], - AggIntent::IRate => vec![ExactKind::IRate], - AggIntent::Increase => vec![ExactKind::Increase], - AggIntent::Avg { .. } => vec![ExactKind::Sum, ExactKind::Count], - _ => vec![], - }; - let exact = if states.is_empty() { - HowToComputeAggregateMeasure::Exact - } else { - HowToComputeAggregateMeasure::ExactAccumulator(states) - }; - let mut choices = ComputationChoices { - candidates: vec![exact], - pruned: vec![], - }; - if let Some(target) = sizing_target(intent) { - if !target.is_exact() { - let (epsilon, delta) = accuracy_budget(&target); - let unit = |value: f64| value > 0.0 && value < 1.0; - if !unit(epsilon) || !unit(delta) { - return Err(PerComputationCandidateError::InvalidAccuracy { - root: None, - measure: None, - }); - } - } - let summaries = summary_candidates(intent).into_iter(); - if matches!(intent, AggIntent::Quantile { q, .. } if !(0.0..=1.0).contains(q)) { - choices.pruned.extend(summaries.map(|algorithm| { - ( - HowToComputeAggregateMeasure::Summary(algorithm), - PruneReason::QuantileOutsideUnitInterval, - ) - })); - } else if target.is_exact() { - choices.pruned.extend(summaries.map(|algorithm| { - ( - HowToComputeAggregateMeasure::Summary(algorithm), - PruneReason::RequiresExact, - ) - })); - } else { - choices - .candidates - .extend(summaries.map(HowToComputeAggregateMeasure::Summary)); - } - } - Ok(choices) -} - -/// Whether an `avg` measure's input may be null, so `count` rows differ from -/// counted values. An unresolvable input is treated as possibly null. -fn nullable_avg_input(node: &OperatorNode, intent: &AggIntent) -> bool { - let Some(NonASAPOp::Aggregate { child, .. }) = node.non_asap() else { - return true; - }; - let column = match intent.input_cols().first() { - Some(column) => Some(*column), - None => resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok(), - }; - column - .and_then(|column| child.schema.fields.get(column)) - .is_none_or(|field| field.nullable) -} - -/// Whether `having` over `aggregate`'s output reads only `by(...)` group keys. -/// A global aggregate (no group keys) never qualifies: its one output row -/// exists even when no input row does, so no row predicate can stand in. -fn key_only_having(aggregate: &OperatorNode, having: &Predicate) -> bool { - let keys = match aggregate.non_asap() { - Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - .. - }) if !by.is_without() => by.len(), - _ => 0, - }; - keys > 0 - && having.0.operator_refs().is_empty() - && having.0.columns_referenced().iter().all(|id| *id < keys) -} - -fn placement(below: bool) -> HavingPlacement { - if below { - HavingPlacement::BelowAggregate - } else { - HavingPlacement::AboveOutput - } -} - -/// Discover one target per measure of each eligible aggregate memo group of -/// `dag`, including operator plans read by scalar roots: Pass 1 walks the -/// groups, so an aggregate several queries read is one target per measure. -/// An aggregate measure is eligible when it has a realization besides exact -/// execution, even if every such realization is pruned; the prune reasons -/// are reported on the target. Measure filters and `HAVING` keep every -/// candidate; the target records how they are applied. A group's readers -/// are its parent groups and the query roots that read it; a consumer -/// `Filter`'s key-only `HAVING` is pushed below only when every reader of -/// the group applies the same predicate, so every candidate of a shared -/// aggregate computes the group's own rows (#1089). -pub fn generate_per_computation_candidates( - dag: &DagOfLogicalMemoGroups, -) -> Result { - let logical_nodes: Rc<[Rc]> = dag - .groups - .iter() - .map(|group| Rc::clone(&group.logical_node)) - .collect(); - let group_of: HashMap<*const OperatorNode, MemoGroupId> = logical_nodes - .iter() - .enumerate() - .map(|(g, node)| (Rc::as_ptr(node), MemoGroupId(g as u32))) - .collect(); - let roots = dag.roots_over_groups(); - let mut seen = HashSet::new(); - // Each aggregate group with the first query that reaches it. - let mut aggregates = Vec::new(); - // Every reader of each group's node: `Some(parent)`, or `None` for a - // query root or scalar reference. - let mut consumers: HashMap<*const OperatorNode, Vec>>> = HashMap::new(); - for (query, root) in roots.iter().enumerate() { - root.validate_structure()?; - let operators = match root { - QueryRoot::Operator(node) => vec![node], - QueryRoot::Scalar(expr) => expr.operator_refs(), - }; - for root in operators { - if root.contains_asap() { - return Err(PerComputationCandidateError::ContainsAsap); - } - consumers.entry(Rc::as_ptr(root)).or_default().push(None); - for node in OperatorNode::reachable(root) { - if !seen.insert(Rc::as_ptr(&node)) { - continue; - } - if node.timing.is_some() || node.materialization != Materialization::NotMaterialized - { - return Err(PerComputationCandidateError::AssignedPhysicalAnnotation); - } - for child in node.children() { - consumers - .entry(Rc::as_ptr(child)) - .or_default() - .push(Some(Rc::clone(&node))); - } - if matches!(node.non_asap(), Some(NonASAPOp::Aggregate { .. })) { - aggregates.push((query, node)); - } - } - } - } - let mut targets = Vec::new(); - let mut pruned_ways_to_compute = Vec::new(); - for (query, node) in aggregates { - let Some(NonASAPOp::Aggregate { - measures, - filters, - having, - .. - }) = node.non_asap() - else { - continue; - }; - let aggregate_group = group_of[&Rc::as_ptr(&node)]; - let mut entries: Vec = having - .iter() - .map(|having| HavingEntry { - filter: None, - predicate: having.clone(), - placement: placement(key_only_having(&node, having)), - }) - .collect(); - let parents = &consumers[&Rc::as_ptr(&node)]; - // Only a `Filter` whose input is this aggregate is its `HAVING`; one - // that reads it from a scalar expression stays a plain consumer. - let mut filters_above: Vec<(&Rc, &Predicate)> = Vec::new(); - for parent in parents.iter().flatten() { - if let Some(NonASAPOp::Filter { child, pred }) = parent.non_asap() { - if Rc::ptr_eq(child, &node) - && !filters_above - .iter() - .any(|(filter, _)| Rc::ptr_eq(filter, parent)) - { - filters_above.push((parent, pred)); - } - } - } - for (filter, predicate) in &filters_above { - // Every reader of the group, across queries, applies it. - let shared_by_all = filters_above.len() == parents.len() - && filters_above.iter().all(|(_, other)| other == predicate); - entries.push(HavingEntry { - filter: Some(Rc::clone(filter)), - predicate: (*predicate).clone(), - placement: placement(shared_by_all && key_only_having(&node, predicate)), - }); - } - for (measure, intent) in measures.iter().enumerate() { - let mut choices = - computation_candidates_for_intent(intent).map_err(|error| match error { - PerComputationCandidateError::InvalidAccuracy { .. } => { - PerComputationCandidateError::InvalidAccuracy { - root: Some(query.to_string()), - measure: Some(measure), - } - } - error => error, - })?; - if choices.candidates == [HowToComputeAggregateMeasure::Exact] - && choices.pruned.is_empty() - { - continue; - } - if matches!(intent, AggIntent::Avg { .. }) && nullable_avg_input(&node, intent) { - choices.prune(PruneReason::NullableAvgInput, |choice| { - matches!(choice, HowToComputeAggregateMeasure::ExactAccumulator(_)) - }); - } - if matches!(intent, AggIntent::Count { .. }) - && node.children()[0].result_kind == OperatorResultKind::Relation - { - choices.prune(PruneReason::NoRelationalCountSketchRealization, |choice| { - matches!(choice, HowToComputeAggregateMeasure::Summary(_)) - }); - } - pruned_ways_to_compute.extend(choices.pruned.into_iter().map( - |(way_to_compute, reason)| PrunedWayToCompute { - target: targets.len(), - of_fused_computation: false, - way_to_compute, - reason, - }, - )); - targets.push(ComputationTarget { - aggregate_group, - measure_index: measure, - ways_to_compute: choices.candidates, - measure_filter: filters.get(measure).cloned().flatten(), - having_filters: entries.clone(), - fused_computation: None, - }); - } - } - add_whole_expression_targets(&mut targets, &logical_nodes); - prune_topk_heaps_over_one_row_per_item( - &mut targets, - &logical_nodes, - &mut pruned_ways_to_compute, - ); - add_nested_sum_targets(&mut targets, &logical_nodes); - close_absorptions(&mut targets); - Ok(PerComputationCandidates { - roots, - logical_nodes, - targets, - pruned_ways_to_compute, - }) -} - -/// Adds to each fused computation's `absorbed_targets` the targets its absorbed -/// targets' own fused computations absorb, transitively: none of them is -/// computed when the fused computation is picked, so enumeration keeps only -/// their first choice ([`PerComputationCandidates::absorbed_choice_varies`]) -/// and [`PerComputationCandidates::combinations`] counts exactly those. -fn close_absorptions(targets: &mut [ComputationTarget]) { - loop { - let mut changed = false; - for t in 0..targets.len() { - let Some(fused) = &targets[t].fused_computation else { - continue; - }; - let beneath: Vec = fused - .absorbed_targets - .iter() - .filter_map(|&a| targets[a].fused_computation.as_ref()) - .flat_map(|inner| inner.absorbed_targets.iter().copied()) - .filter(|b| !fused.absorbed_targets.contains(b)) - .collect(); - if let Some(fused) = targets[t] - .fused_computation - .as_mut() - .filter(|_| !beneath.is_empty()) - { - for b in beneath { - if !fused.absorbed_targets.contains(&b) { - fused.absorbed_targets.push(b); - } - } - changed = true; - } - } - if !changed { - return; - } - } -} - -/// Nested-sum merge: a grouped `sum` over the value of a `sum` target -/// beneath it gets a fused computation ([`ComputationTarget::fused_computation`]) -/// with the one candidate `ExactAccumulator([Sum])`: one exact `Sum` -/// accumulator over the inner sum's input, grouped by the outer keys (the -/// sum of the inner groups' sums is the sum of their rows), absorbing the -/// inner target. Offered whatever else reads the inner sum: it is one more -/// candidate of the outer sum's group, and the inner sum's group keeps its -/// own candidates for its other readers. Applied when neither has a filter -/// or `HAVING` -/// ([`nested_sum_input`]), and when the outer sum is not itself absorbed by -/// a merge and has no fused computation yet, and the inner sum has none, so -/// merges do not stack. An outer sum a whole-expression top-k absorbs -/// (`topk(k, sum by (G) (sum_over_time(m[W])))`) keeps its merge for the -/// top-k's own candidates. -fn add_nested_sum_targets(targets: &mut [ComputationTarget], logical_nodes: &[Rc]) { - let aggregate = |target: &ComputationTarget| &logical_nodes[target.aggregate_group.0 as usize]; - // The targets a merge absorbs. - let mut absorbed: HashSet = HashSet::new(); - for t in 0..targets.len() { - let target = &targets[t]; - if absorbed.contains(&t) - || target.fused_computation.is_some() - || !target.having_filters.is_empty() - { - continue; - } - let Some(NonASAPOp::Aggregate { child, .. }) = aggregate(target).non_asap() else { - continue; - }; - let Some(inner) = targets - .iter() - .position(|other| Rc::ptr_eq(aggregate(other), child)) - else { - continue; - }; - // Merges do not stack: the inner sum must not have a fused - // computation of its own. - if !targets[inner].having_filters.is_empty() - || targets[inner].fused_computation.is_some() - || nested_sum_input(aggregate(target)).is_none() - { - continue; - } - absorbed.insert(inner); - let merge = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); - targets[t].fused_computation = Some(FusedComputation { - absorbed_targets: vec![inner], - ways_to_compute: vec![merge], - }); - } -} - -/// The input rows, the inner sum and the outer grouping (moved onto the -/// input rows by column, [`Reduction::partitions_over`]) of a nested-sum merge of -/// `target`, or `None` when it does not apply: `target` is a grouped `sum` -/// (not by `without`) of the value column of a per-series or grouped `sum` -/// directly beneath it, neither filtered nor with `HAVING`, and not grouped -/// by the metric name a per-series inner sum drops. -fn nested_sum_input(target: &OperatorNode) -> Option<(Rc, AggIntent, Reduction)> { - use asap_types::ir::operator::non_asap::any_measure_filtered; - let Some(NonASAPOp::Aggregate { - child: inner, - reduction: reduction @ Reduction::Reduce(by), - measures, - filters, - having: None, - .. - }) = target.non_asap() - else { - return None; - }; - let [AggIntent::Sum { col }] = measures.as_slice() else { - return None; - }; - let Some(NonASAPOp::Aggregate { - child: rows, - reduction: inner_reduction, - measures: inner_measures, - filters: inner_filters, - having: None, - .. - }) = inner.non_asap() - else { - return None; - }; - let [inner_sum @ AggIntent::Sum { .. }] = inner_measures.as_slice() else { - return None; - }; - if by.is_without() || any_measure_filtered(filters) || any_measure_filtered(inner_filters) { - return None; - } - // The inner sum's value column: a per-series sum keeps its labels and - // carries the value as the sample value; a grouped one ends with it. - let value = match inner_reduction { - Reduction::PerEntity => resolve_column_ref(&ColumnRef::SampleValue, &inner.schema).ok()?, - Reduction::Reduce(keys) if !keys.is_without() => keys.len(), - Reduction::Reduce(_) => return None, - }; - let summed = match col { - None => resolve_column_ref(&ColumnRef::SampleValue, &inner.schema).ok()?, - Some(column) => *column, - }; - if summed != value || by.keys().contains(&value) { - return None; - } - // A per-series sum drops the metric name, which the raw rows keep: - // grouping them by it would name groups the nested sums leave unnamed. - if matches!(inner_reduction, Reduction::PerEntity) - && by.keys().iter().any(|&key| { - inner - .schema - .fields - .get(key) - .is_some_and(|f| f.name == "__name__") - }) - { - return None; - } - let partitions = reduction.partitions_over(&inner.schema, &rows.schema)?; - Some((Rc::clone(rows), inner_sum.clone(), partitions)) -} - -/// Whole-expression top-k (Example 1's Q2): a top-k over a per-item `sum` -/// or `count` that is itself a target gets a fused computation -/// ([`ComputationTarget::fused_computation`]) absorbing the inner target, whose -/// candidates are the top-k's heap summaries (Count-Min and CountSketch -/// with a heap), each one heap over the inner aggregate's input, keyed by -/// the ranked item and weighted by the summed value. Those heaps over the -/// exact per-item result leave the top-k's own candidates: the fused -/// computation replaces them. Over a grouped sum of per-series sums or -/// counts of raw data samples, the heap reads those samples and absorbs the -/// per-series target too. Offered whatever else reads the inner aggregate: -/// it is one more candidate of the top-k's group, and the absorbed groups -/// keep their own candidates for their other readers. Applied when neither -/// has a filter or `HAVING`, and when the keyed-additive rule realizes it -/// ([`whole_expression_input`]). -fn add_whole_expression_targets( - targets: &mut [ComputationTarget], - logical_nodes: &[Rc], -) { - let aggregate = |target: &ComputationTarget| &logical_nodes[target.aggregate_group.0 as usize]; - for t in 0..targets.len() { - let target = &targets[t]; - let Some(NonASAPOp::Aggregate { child, .. }) = aggregate(target).non_asap() else { - continue; - }; - let Some(inner) = targets - .iter() - .position(|other| Rc::ptr_eq(aggregate(other), child)) - else { - continue; - }; - if target.measure_filter.is_some() - || !target.having_filters.is_empty() - || targets[inner].measure_filter.is_some() - || !targets[inner].having_filters.is_empty() - { - continue; - } - let Some((input, ..)) = whole_expression_input(aggregate(target)) else { - continue; - }; - // A heap over the raw data samples beneath a grouped sum of per-series - // sums or counts also absorbs the per-series target. - let mut absorbs = vec![inner]; - if let Some(NonASAPOp::Aggregate { - child: per_series, .. - }) = aggregate(&targets[inner]).non_asap() - { - let reads_below = per_series - .children() - .iter() - .any(|&rows| Rc::ptr_eq(rows, &input)); - if reads_below { - absorbs.extend( - targets - .iter() - .position(|other| Rc::ptr_eq(aggregate(other), per_series)), - ); - } - } - let (heaps, own): (Vec<_>, Vec<_>) = - targets[t] - .ways_to_compute - .iter() - .cloned() - .partition(|candidate| { - matches!(candidate, HowToComputeAggregateMeasure::Summary(family) - if family.capability().answers(StatisticKind::TopK)) - }); - if heaps.is_empty() { - continue; - } - let target = &mut targets[t]; - target.ways_to_compute = own; - target.fused_computation = Some(FusedComputation { - absorbed_targets: absorbs, - ways_to_compute: heaps, - }); - } -} - -/// Prunes the own heap candidates of each PromQL top-k -/// ([`PruneReason::TopKHeapOverOneRowPerItem`]); a fused whole-expression -/// heap ([`add_whole_expression_targets`]) has already moved out of them. -fn prune_topk_heaps_over_one_row_per_item( - targets: &mut [ComputationTarget], - logical_nodes: &[Rc], - pruned_ways_to_compute: &mut Vec, -) { - for (index, target) in targets.iter_mut().enumerate() { - let Some(NonASAPOp::Aggregate { - measures, child, .. - }) = logical_nodes[target.aggregate_group.0 as usize].non_asap() - else { - continue; - }; - if !matches!( - measures.get(target.measure_index), - Some(AggIntent::TopK { .. }) - ) { - continue; - } - // A PromQL top-k ranks an instant vector: one row per series or - // group. (A SQL top-k keeps its heaps.) - if !reads_only_time_series(child) { - continue; - } - let pruned = prune_own(&mut target.ways_to_compute, |candidate| { - matches!(candidate, HowToComputeAggregateMeasure::Summary(family) - if family.capability().answers(StatisticKind::TopK)) - }); - pruned_ways_to_compute.extend(pruned.into_iter().map(|way_to_compute| { - PrunedWayToCompute { - target: index, - of_fused_computation: false, - way_to_compute, - reason: PruneReason::TopKHeapOverOneRowPerItem, - } - })); - } -} - -/// Whether every scan below `node` reads a time series (PromQL rows). -fn reads_only_time_series(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), - _ => node.children().iter().all(|c| reads_only_time_series(c)), - } -} - -/// The input, update and partitions of a whole-expression top-k over -/// `target`'s inner aggregate, by the keyed-additive rule, or `None` when it -/// does not apply. Rows that carry the full series identity rank it as a -/// column, as [`summary_update`] does. The partitions are `target`'s, moved -/// onto the input the heap reads ([`Reduction::partitions_over`]). -fn whole_expression_input( - target: &OperatorNode, -) -> Option<(Rc, SummaryUpdate, Reduction)> { - use crate::pass1::realization::realize_keyed_additive_summary_input; - let Some(NonASAPOp::Aggregate { - child, - reduction, - measures, - filters, - having: None, - .. - }) = target.non_asap() - else { - return None; - }; - let [intent @ AggIntent::TopK { .. }] = measures.as_slice() else { - return None; - }; - if asap_types::ir::operator::non_asap::any_measure_filtered(filters) { - return None; - } - // The rule reads only the family's algorithm, and rejects Count-Min over - // weights it cannot prove non-negative. Ask with CountSketch so one - // update serves both heap families; plan selection decides whether - // Count-Min is admissible, as for every other Count-Min candidate. - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - asap_types::ir::schema::SketchParams::CountSketchWithHeap { - width: 1, - depth: 1, - heap_size: 1, - }, - ), - GroupingStrategy::default(), - ); - let realized = realize_keyed_additive_summary_input(intent, &family, reduction, child)?; - let mut input = realized.input; - let per_series = matches!( - child.non_asap(), - Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - .. - }) - ); - let partitions = reduction.partitions_over(&child.schema, &realized.child.schema)?; - if per_series && realized.child.schema.has_promql_series_identity() { - input.item = Some(asap_types::ir::schema::topk::series_identity_item( - &realized.child.schema, - partitions.group_keys().map_or(&[][..], |keys| keys.keys()), - )); - } else if realized.child.schema.closed - && matches!(input.item, Some(SummaryInputExpr::EntityIdentity(_))) - { - // An encoded label set needs an open PromQL schema. PromQL rows that - // carry the series identity are closed, but their inner group labels - // are item columns. SQL group keys are explicit columns too, so their - // keyed heap can consume raw rows without a per-key aggregate first. - return None; - } - Some((realized.child, input, partitions)) -} - -/// `update` with the open requirement that its weights are samples of -/// `metric` ([`WeightDomain::RawDataSamplesOfMetric`]), when it is a keyed heap -/// update whose weights are not otherwise proven. -fn with_raw_data_samples_requirement( - mut update: SummaryUpdate, - metric: Option<&str>, -) -> SummaryUpdate { - if let Some(metric) = metric { - if update.item.is_some() && update.weight_domain == WeightDomain::UnknownOrSigned { - update.weight_domain = WeightDomain::RawDataSamplesOfMetric { - metric: metric.to_string(), - }; - } - } - update -} - -/// Build one whole-workload candidate (end of Pass 1): `choice[i]` picks -/// `inventory.targets[i]`'s candidate, or past its own candidates its fused -/// computation's ([`ComputationTarget::computation_for`]). An aggregate with -/// a chosen candidate other than `Exact` is replaced measure by measure: an -/// exact accumulator by `SummaryAgg` → `FinalizeExactAccumulator`, a summary -/// by `SummaryAgg` → `SummaryEstimate`, `avg` by its `sum` and `count` -/// accumulators divided, a fused computation by one state over the absorbed -/// target's input, and any other measure by its own single-measure -/// aggregate; several measures are joined back on the group keys. A summary -/// is sized here for its measure's accuracy requirement; Pass 1 leaves it -/// unsized; several approximate inputs of one query share its budget -/// equally (`accuracy_budget_parts_per_measure`). A measure filter becomes `SummaryAgg.filter`; -/// `HAVING` is applied as its target records. A sub-DAG shared between roots -/// stays shared. Every node carries the guarantee `asap_accuracy::propagation::composed_guarantee` -/// gives it. -#[cfg(any(test, feature = "test-support"))] -pub fn compose_logical_candidate( - inventory: &PerComputationCandidates, - choice: &[usize], -) -> Result, PerComputationCandidateError> { - compose_with_accuracy_budget_parts(inventory, &inventory.roots, choice, None) -} - -/// [`compose_logical_candidate`] with every sketch summary sized for one of -/// `accuracy_budget_parts` equal parts of its measure's budget, whatever the -/// roots compose: the size a summary gets when it is one of -/// `accuracy_budget_parts` approximate inputs of a query -/// ([`accuracy_budget_parts_per_measure`]). `accuracy_budget_parts` 1 sizes -/// for the whole budget. The sub-expression memo composes one aggregate -/// alone this way, once per number of parts its readers may give it. -pub(crate) fn compose_logical_candidate_with_accuracy_budget_parts( - inventory: &PerComputationCandidates, - choice: &[usize], - accuracy_budget_parts: usize, -) -> Result, PerComputationCandidateError> { - compose_with_accuracy_budget_parts( - inventory, - &inventory.roots, - choice, - Some(accuracy_budget_parts), - ) -} - -/// [`compose_logical_candidate`] of `roots` (some of `inventory.roots`), -/// each summary sized for one of `accuracy_budget_parts` equal parts of its -/// budget, or as -/// [`accuracy_budget_parts_per_measure`] splits it when `None`. -fn compose_with_accuracy_budget_parts( - inventory: &PerComputationCandidates, - roots: &[QueryRoot], - choice: &[usize], - accuracy_budget_parts: Option, -) -> Result, PerComputationCandidateError> { - if choice.len() != inventory.targets.len() { - return Err(PerComputationCandidateError::InvalidChoice); - } - let mut chosen: Chosen = HashMap::new(); - for (target, &index) in inventory.targets.iter().zip(choice) { - let (candidate, absorbs) = target - .computation_for(index) - .ok_or(PerComputationCandidateError::InvalidChoice)?; - chosen - .entry(Rc::as_ptr(inventory.aggregate(target))) - .or_default() - .push(ChosenComputation { - target, - candidate, - fused: absorbs.is_some(), - }); - } - // Consumer `Filter`s whose key-only `HAVING` moves below a substituted - // aggregate: each is replaced by its input. - let pushed_below: HashSet<_> = chosen - .values() - .filter(|targets| substituted(targets)) - .flat_map(|targets| &targets[0].target.having_filters) - .filter(|entry| entry.placement == HavingPlacement::BelowAggregate) - .filter_map(|entry| entry.filter.as_ref().map(Rc::as_ptr)) - .collect(); - // Each summary is first sized for its measure's whole budget. A query - // whose value composes several approximate inputs is then composed - // again with each sized for an equal share ([`accuracy_budget_parts_per_measure`]). - let whole = Sizing::default(); - let composed = compose_roots(roots, &chosen, &pushed_below, &whole)?; - let shares = match accuracy_budget_parts { - None => accuracy_budget_parts_per_measure(&composed, &whole.built.into_inner()), - Some(share) if share > 1 => whole - .built - .into_inner() - .into_iter() - .map(|(_, key)| (key, share)) - .collect(), - Some(_) => HashMap::new(), - }; - if shares.is_empty() { - return Ok(composed); - } - let shared = Sizing { - shares, - ..Sizing::default() - }; - compose_roots(roots, &chosen, &pushed_below, &shared) -} - -/// One measure of an aggregate: the aggregate and the measure's index. -type MeasureKey = (*const OperatorNode, usize); - -/// How [`compose_logical_candidate`] sizes summaries in one composition. -#[derive(Default)] -struct Sizing { - /// How many approximate inputs each measure's summary shares its - /// query's budget with; a measure not listed has the whole budget. - shares: HashMap, - /// Every sketch summary state built, with the measure it sizes. - built: std::cell::RefCell, MeasureKey)>>, -} - -/// The equal budget split among the approximate inputs of each query -/// (#669b): a query whose value carries a guarantee that is not exact -/// composes every sketch summary state below its root -/// ([`approximate_inputs`]), so with `n ≥ 2` of them each is sized for one -/// of `n` equal shares of the query's target. A state several queries read -/// takes the largest `n`. Only states `built` lists (by the measure they -/// size) are split. Empty when no query composes two approximate inputs. -fn accuracy_budget_parts_per_measure( - roots: &[QueryRoot], - built: &[(Rc, MeasureKey)], -) -> HashMap { - let measure: HashMap<_, _> = built - .iter() - .map(|(state, key)| (Rc::as_ptr(state), *key)) - .collect(); - let mut shares = HashMap::new(); - for root in roots { - let inputs = approximate_inputs(root); - if inputs.len() < 2 { - continue; - } - for state in &inputs { - if let Some(key) = measure.get(&Rc::as_ptr(state)) { - let share = shares.entry(*key).or_insert(1); - *share = inputs.len().max(*share); - } - } - } - shares -} - -/// Every root of `roots` with the chosen candidates substituted, each -/// summary sized as `sizing` says, and every node with its guarantee. -fn compose_roots( - roots: &[QueryRoot], - chosen: &Chosen<'_>, - pushed_below: &HashSet<*const OperatorNode>, - sizing: &Sizing, -) -> Result, PerComputationCandidateError> { - let (mut cache, mut guaranteed) = (HashMap::new(), HashMap::new()); - let mut rewrite = |node: &Rc| { - let node = rewrite(node, chosen, pushed_below, sizing, &mut cache)?; - with_guarantees(&node, &mut guaranteed).map_err(PerComputationCandidateError::from) - }; - let roots = roots - .iter() - .map(|root| { - let root = match root { - QueryRoot::Operator(node) => QueryRoot::Operator(rewrite(node)?), - QueryRoot::Scalar(expr) => { - let mut rewritten = HashMap::new(); - for node in expr.operator_refs() { - rewritten.insert(Rc::as_ptr(node), rewrite(node)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| rewritten[&Rc::as_ptr(node)].clone()), - ) - } - }; - // A substitution this module cannot express is an error, never an - // invalid DAG. - root.validate_structure()?; - Ok(root) - }) - .collect::, PerComputationCandidateError>>()?; - // The built states, as the roots hold them: with their guarantees. - for (state, _) in sizing.built.borrow_mut().iter_mut() { - if let Some(with_guarantee) = guaranteed.get(&Rc::as_ptr(state)) { - *state = with_guarantee.clone(); - } - } - Ok(roots) -} - -/// One target's chosen computation in a whole-workload choice. -struct ChosenComputation<'a> { - /// The target whose measure it computes. - target: &'a ComputationTarget, - /// How it is computed. - candidate: &'a HowToComputeAggregateMeasure, - /// Whether the candidate is the target's fused computation's, which - /// absorbs the targets beneath; `false` for one of its own candidates. - fused: bool, -} - -/// Each aggregate's targets with their chosen computations. -type Chosen<'a> = HashMap<*const OperatorNode, Vec>>; -type NodeToRewrittenNodeCache = HashMap<*const OperatorNode, Rc>; - -/// Whether any measure of the aggregate is substituted by something other than -/// its plain exact computation. -fn substituted(targets: &[ChosenComputation<'_>]) -> bool { - targets - .iter() - .any(|chosen| *chosen.candidate != HowToComputeAggregateMeasure::Exact) -} - -fn rewrite( - node: &Rc, - chosen: &Chosen<'_>, - pushed_below: &HashSet<*const OperatorNode>, - sizing: &Sizing, - cache: &mut NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - rewrite(child, chosen, pushed_below, sizing, cache)?; - } - let changed = node - .children() - .iter() - .any(|child| !Rc::ptr_eq(child, &cache[&Rc::as_ptr(child)])); - let rebuilt = match (chosen.get(&Rc::as_ptr(node)), node.non_asap()) { - (Some(targets), _) if substituted(targets) => substitute(node, targets, sizing, cache)?, - (_, Some(NonASAPOp::Filter { child, .. })) if pushed_below.contains(&Rc::as_ptr(node)) => { - cache[&Rc::as_ptr(child)].clone() - } - _ if changed => Rc::new(node.with_new_children(|child| cache[&Rc::as_ptr(child)].clone())?), - _ => node.clone(), - }; - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - Ok(rebuilt) -} - -/// Replace an aggregate whose measures are `targets`' (or exact, for a -/// measure that is no target) by its substitution, with the same output -/// columns. -fn substitute( - target: &Rc, - targets: &[ChosenComputation<'_>], - sizing: &Sizing, - cache: &NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { - child, - reduction, - measures, - filters, - .. - }) = target.non_asap() - else { - return Err(PerComputationCandidateError::Unsupported( - "target is not an aggregate", - )); - }; - // A key-only `HAVING` is a row predicate over the input: rows of a - // dropped group never reach the state. - let having = &targets[0].target.having_filters; - // Operator refs in a predicate (a subquery) are this aggregate's - // children, already rewritten. - let rewritten = |pred: &Predicate| { - Predicate( - pred.0 - .map_operator_refs(&mut |node| cache[&Rc::as_ptr(node)].clone()), - ) - }; - // The metric whose raw data samples, or plain sums of them, are every - // value the aggregate reads: a heap-sketch weight read from them carries - // the open requirement `WeightDomain::RawDataSamplesOfMetric`, which - // plan selection confirms. - let raw_data_samples = raw_data_samples_of_metric(child); - let mut input = cache[&Rc::as_ptr(child)].clone(); - for entry in having { - if entry.placement == HavingPlacement::BelowAggregate { - let keys = reduction.group_keys().map_or(&[][..], |by| by.keys()); - let pred = Predicate(entry.predicate.0.map_columns(&mut |column| keys[column])); - input = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred, - child: input, - }))?; - } - } - let parts = measures - .iter() - .enumerate() - .map(|(measure, intent)| { - // `fused`: the candidate is the target's fused computation's, - // which absorbs the target beneath. - let (candidate, raw_data_samples_of, fused) = targets - .iter() - .find(|chosen| chosen.target.measure_index == measure) - .map_or( - (&HowToComputeAggregateMeasure::Exact, None, false), - |chosen| (chosen.candidate, raw_data_samples.as_deref(), chosen.fused), - ); - let filter = filters.get(measure).and_then(Option::as_ref).map(rewritten); - let key = (Rc::as_ptr(target), measure); - if fused { - return match candidate { - HowToComputeAggregateMeasure::ExactAccumulator(_) => nested_sum(target, cache), - HowToComputeAggregateMeasure::Summary(algorithm) => whole_expression( - target, - intent, - algorithm, - raw_data_samples_of, - (sizing, key), - cache, - ), - HowToComputeAggregateMeasure::Exact => { - Err(PerComputationCandidateError::Unsupported( - "plain exact execution of a fused computation", - )) - } - }; - } - substitute_measure( - target, - intent, - candidate, - filter, - &input, - raw_data_samples_of, - (sizing, key), - ) - }) - .collect::, _>>()?; - let mut output = match <[_; 1]>::try_from(parts) { - Ok([part]) => with_names_of(target, part)?, - Err(parts) => join_measures(target, parts)?, - }; - for entry in having { - if entry.placement == HavingPlacement::AboveOutput && entry.filter.is_none() { - output = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: rewritten(&entry.predicate), - child: output, - }))?; - } - } - Ok(output) -} - -/// `output` with `target`'s column names: an estimate or accumulator names -/// its value after the operation (`quantile`, `sum`), not the aggregate's -/// output column. -fn with_names_of( - target: &OperatorNode, - output: Rc, -) -> Result, PerComputationCandidateError> { - let names = |node: &OperatorNode| -> Vec { - node.schema - .fields - .iter() - .map(|field| field.name.clone()) - .collect() - }; - // A top-k's ranked rows may carry their item columns after the - // aggregate's ([`ranked_as_topk`]). - if names(&output).starts_with(&names(target)) { - return Ok(with_aggregate_qualifiers(target, output)); - } - if output.schema.fields.len() != target.schema.fields.len() { - return Err(PerComputationCandidateError::Unsupported( - "substituted output with other columns than the aggregate", - )); - } - let cols = names(target) - .into_iter() - .enumerate() - .map(|(column, name)| ProjectItem { - alias: Some(name), - expr: ScalarExpr::Column(column), - }) - .collect(); - Ok(with_aggregate_qualifiers( - target, - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: output, - }))?, - )) -} - -/// An equivalent aggregate replacement must retain SQL field identity. These -/// explicit qualifiers survive child rebinding; types and keys remain derived -/// from executable operators rather than copied from the requested output. -fn with_aggregate_qualifiers(target: &OperatorNode, output: Rc) -> Rc { - if output - .schema - .fields - .iter() - .zip(&target.schema.fields) - .all(|(field, expected)| field.table == expected.table) - { - return output; - } - let mut output = output.as_ref().clone(); - for (field, expected) in output.schema.fields.iter_mut().zip(&target.schema.fields) { - field.table = expected.table.clone(); - } - Rc::new(output) -} - -/// A whole-expression top-k for `target` ([`whole_expression_input`]): one -/// heap summary of `algorithm` over the absorbed aggregate's rewritten -/// input, partitioned as `target`. -fn whole_expression( - target: &OperatorNode, - intent: &AggIntent, - algorithm: &SketchAlgorithm, - raw_data_samples_of: Option<&str>, - (sizing, key): (&Sizing, MeasureKey), - cache: &NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - let (input, update, partitions) = whole_expression_input(target).ok_or( - PerComputationCandidateError::Unsupported("whole-expression top-k input"), - )?; - let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: cache[&Rc::as_ptr(&input)].clone(), - input: with_raw_data_samples_requirement(update, raw_data_samples_of), - family: FieldDataType::Sketch( - sized(algorithm, intent, sizing.share(key)), - GroupingStrategy::default(), - ), - reduction: partitions, - grouping: GroupingStrategy::default(), - filter: None, - }))?; - sizing.built.borrow_mut().push((state.clone(), key)); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: state, - query: sketch_statistic(intent).ok_or(PerComputationCandidateError::Unsupported( - "sketch evaluation", - ))?, - }))?; - ranked_as_topk(target, estimate) -} - -/// A heap's top-k `estimate` (`[partitions.., items.., topk]`, one row per -/// ranked item) in `target`'s columns: its `[keys.., topk_k]`, with -/// `topk_k` the estimated Float64 weight, then the item columns. This is -/// the exact top-k's physical shape (its ranked rows' value, then the -/// columns naming each ranked item: PromQL labels, SQL group keys), so -/// readers of `[keys.., topk_k]` find those columns where they were. -fn ranked_as_topk( - target: &OperatorNode, - estimate: Rc, -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { reduction, .. }) = target.non_asap() else { - return Err(PerComputationCandidateError::Unsupported( - "top-k target is not an aggregate", - )); - }; - let ranked_index = reduction.group_keys().map_or(0, |keys| keys.len()); - let fields = &estimate.schema.fields; - let estimated = fields.len() - 1; - let cols = target - .schema - .fields - .iter() - .enumerate() - .map(|(index, field)| { - let source = if index == ranked_index { - estimated - } else { - fields[..estimated] - .iter() - .position(|input| input.name == field.name) - .ok_or(PerComputationCandidateError::Unsupported( - "top-k estimate without a ranked-row identity column", - ))? - }; - Ok(ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(source), - }) - }) - .collect::, PerComputationCandidateError>>()?; - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: estimate, - }, - ))?) -} - -/// A nested-sum merge for `target` ([`nested_sum_input`]): one exact `Sum` -/// accumulator over the absorbed sum's rewritten input, grouped as -/// `target`, finalized. -fn nested_sum( - target: &OperatorNode, - cache: &NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - let (rows, intent, reduction) = nested_sum_input(target).ok_or( - PerComputationCandidateError::Unsupported("nested-sum merge input"), - )?; - let state = exact_accumulator( - ExactKind::Sum, - &intent, - &cache[&Rc::as_ptr(&rows)], - reduction, - )?; - Ok(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child: state }, - ))?) -} - -/// An unfiltered exact accumulator of `kind` for `intent` over `input`, -/// grouped by `reduction`. -pub(crate) fn exact_accumulator( - kind: ExactKind, - intent: &AggIntent, - input: &Rc, - reduction: Reduction, -) -> Result, PerComputationCandidateError> { - let family = FieldDataType::exact(kind); - let read = measure_input(intent, input, None)?; - Ok(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::SummaryAgg { - input: summary_update(intent, &family, &reduction, &input.schema, read)?, - child: Rc::clone(input), - family, - reduction, - grouping: GroupingStrategy::default(), - filter: None, - }, - ))?) -} - -/// One measure of `target` over `input`: its group keys, then its value. -fn substitute_measure( - target: &OperatorNode, - intent: &AggIntent, - candidate: &HowToComputeAggregateMeasure, - filter: Option, - input: &Rc, - raw_data_samples_of: Option<&str>, - (sizing, key): (&Sizing, MeasureKey), -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { - reduction, - output_names, - .. - }) = target.non_asap() - else { - unreachable!("substitute checked the aggregate") - }; - let state = |family: FieldDataType, intent: &AggIntent, read: SummaryInputExpr| { - Ok::<_, PerComputationCandidateError>(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::SummaryAgg { - child: input.clone(), - input: with_raw_data_samples_requirement( - summary_update(intent, &family, reduction, &input.schema, read)?, - raw_data_samples_of, - ), - family, - reduction: reduction.clone(), - grouping: GroupingStrategy::default(), - filter: filter.clone(), - }, - ))?) - }; - let finalized = |kind: &ExactKind, intent: &AggIntent, read| { - let state = state(FieldDataType::exact(*kind), intent, read)?; - Ok::<_, PerComputationCandidateError>(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child: state }, - ))?) - }; - Ok(match candidate { - HowToComputeAggregateMeasure::Exact => { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: input.clone(), - reduction: reduction.clone(), - measures: vec![intent.clone()], - // One name per measure, as the target names this one (an - // empty name is the intent's synthetic one). - output_names: vec![output_names.get(key.1).cloned().unwrap_or_default()], - filters: vec![filter.clone()], - having: None, - }))? - } - HowToComputeAggregateMeasure::ExactAccumulator(kinds) if kinds.len() == 1 => { - let read = measure_input(intent, input, filter.as_ref())?; - finalized(&kinds[0], intent, read)? - } - HowToComputeAggregateMeasure::Summary(algorithm) => { - let family = FieldDataType::Sketch( - sized(algorithm, intent, sizing.share(key)), - GroupingStrategy::default(), - ); - let summary_input = state( - family, - intent, - measure_input(intent, input, filter.as_ref())?, - )?; - sizing.built.borrow_mut().push((summary_input.clone(), key)); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query: sketch_statistic(intent).ok_or( - PerComputationCandidateError::Unsupported("sketch evaluation"), - )?, - }))?; - match intent { - AggIntent::TopK { .. } => ranked_as_topk(target, estimate)?, - _ => estimate, - } - } - HowToComputeAggregateMeasure::ExactAccumulator(kinds) => { - let (AggIntent::Avg { col }, [ExactKind::Sum, ExactKind::Count]) = - (intent, kinds.as_slice()) - else { - return Err(PerComputationCandidateError::Unsupported( - "several exact states other than avg's sum then count", - )); - }; - let sum_intent = AggIntent::Sum { col: *col }; - // The count counts the averaged column, as `avg` does: it is - // non-null (a nullable input prunes this candidate), so it counts - // the rows the sum reads. - let read = measure_input(&sum_intent, input, filter.as_ref())?; - let sum = finalized(&ExactKind::Sum, &sum_intent, read.clone())?; - let count = finalized( - &ExactKind::Count, - &AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, - read, - )?; - if input.result_kind == OperatorResultKind::Relation { - return sql_avg(reduction, filter.is_some(), sum, count); - } - // `avg` is Float64 whatever the summed type, as `Avg`'s output is, - // and a PromQL binary operator reads Float64 sample values. A - // grouped sum or count ends with its value, cast when not Float64 - // (a count is Int64). A per-series sum keeps the input's labels, - // so its value is the sample-value column; a per-series count is - // Float64. - let (sum, count) = match reduction { - Reduction::Reduce(by) => ( - last_as_float64(by.len(), sum)?, - last_as_float64(by.len(), count)?, - ), - Reduction::PerEntity => { - let value = resolve_column_ref(&ColumnRef::SampleValue, &sum.schema) - .ok() - .and_then(|column| sum.schema.fields.get(column)); - if value.map(|field| &field.dtype) - != Some(&FieldDataType::Plain(DataType::Float64)) - { - return Err(PerComputationCandidateError::Unsupported( - "non-Float64 per-series avg", - )); - } - (sum, count) - } - }; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - // A per-series window may hold no finite sum to divide. - checked_finite_division: matches!(reduction, Reduction::PerEntity), - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), - vector_match: None, - }, - return_bool: false, - lhs: sum, - rhs: count, - }))? - } - }) -} - -/// `node`, a grouped aggregate's `keys` group keys and then its value, with -/// the value cast to Float64 unless it already is one. -fn last_as_float64( - keys: usize, - node: Rc, -) -> Result, PerComputationCandidateError> { - if node.schema.fields.last().map(|field| &field.dtype) - == Some(&FieldDataType::Plain(DataType::Float64)) - { - return Ok(node); - } - let mut cols = key_items(keys); - cols.push(ProjectItem { - alias: None, - expr: ScalarExpr::Cast { - expr: Box::new(ScalarExpr::Column(keys)), - to: DataType::Float64, - try_cast: false, - }, - }); - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: node, - }, - ))?) -} - -/// SQL `avg` from its finalized `sum` and `count`: joined on the group keys -/// (NULL keys match), then `sum / count` as Float64. A group with no row -/// passing the measure filter, or an empty global input, counts 0 and its -/// `avg` is NULL. -fn sql_avg( - reduction: &Reduction, - filtered: bool, - sum: Rc, - count: Rc, -) -> Result, PerComputationCandidateError> { - let Reduction::Reduce(by) = reduction else { - return Err(PerComputationCandidateError::Unsupported( - "SQL avg without `by` keys", - )); - }; - if by.is_without() { - return Err(PerComputationCandidateError::Unsupported( - "SQL avg without `by` keys", - )); - } - let keys = by.len(); - let float = |column| { - Box::new(ScalarExpr::Cast { - expr: Box::new(ScalarExpr::Column(column)), - to: DataType::Float64, - try_cast: false, - }) - }; - let count_column = keys + 1 + keys; - let mut avg = ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Div, - left: float(keys), - right: float(count_column), - semantics: ExprSemantics::Sql, - }; - if filtered || keys == 0 { - avg = ScalarExpr::Case { - operand: None, - branches: vec![( - ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(count_column)), - op: CompareOpKind::Eq, - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), - semantics: ExprSemantics::Sql, - }, - ScalarExpr::Literal(ScalarValue::Null), - )], - else_expr: Some(Box::new(avg)), - }; - } - let mut cols = key_items(keys); - cols.push(ProjectItem { - alias: None, - expr: avg, - }); - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: join_on_keys(keys, sum, count)?, - }, - ))?) -} - -/// `left` joined with `right` on their leading `keys` columns; NULL keys -/// match each other, as grouping treats them. -fn join_on_keys( - keys: usize, - left: Rc, - right: Rc, -) -> Result, PerComputationCandidateError> { - let width = left.schema.fields.len(); - let column = |id| Box::new(ScalarExpr::Column(id)); - let matches = (0..keys) - .map(|key| { - let equal = ScalarExpr::Compare { - left: column(key), - op: CompareOpKind::Eq, - right: column(width + key), - semantics: ExprSemantics::Sql, - }; - if right.schema.fields[key].nullable { - ScalarExpr::BoolOr(vec![ - equal, - ScalarExpr::BoolAnd(vec![ - ScalarExpr::IsNull(column(key)), - ScalarExpr::IsNull(column(width + key)), - ]), - ]) - } else { - equal - } - }) - .collect::>(); - let (kind, pred) = match matches.len() { - 0 => ( - JoinKind::Cross, - ScalarExpr::Literal(ScalarValue::Boolean(true)), - ), - _ => (JoinKind::Inner, ScalarExpr::BoolAnd(matches)), - }; - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Join { - kind, - pred: Predicate(pred), - left, - right, - }, - ))?) -} - -/// Join single-measure `parts` (each: group keys, then one value) on their -/// group keys and project `target`'s output columns. Every part has the same -/// groups, so an inner join keeps them all; NULL keys match each other, as -/// grouping treats them. -fn join_measures( - target: &OperatorNode, - parts: Vec>, -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - .. - }) = target.non_asap() - else { - return Err(PerComputationCandidateError::Unsupported( - "multi-measure aggregate without `by` keys", - )); - }; - if by.is_without() { - return Err(PerComputationCandidateError::Unsupported( - "multi-measure aggregate without `by` keys", - )); - } - let keys = by.len(); - let width = keys + 1; - let mut parts = parts.into_iter(); - let mut joined = parts.next().expect("an aggregate has measures"); - for part in parts { - joined = join_on_keys(keys, joined, part)?; - } - let mut cols = key_items(keys); - cols.extend( - (0..target.schema.fields.len() - keys).map(|measure| ProjectItem { - alias: None, - expr: ScalarExpr::Column(measure * width + keys), - }), - ); - for (item, field) in cols.iter_mut().zip(&target.schema.fields) { - item.alias = Some(field.name.clone()); - } - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: joined, - }, - ))?) -} - -fn key_items(keys: usize) -> Vec { - (0..keys) - .map(|key| ProjectItem { - alias: None, - expr: ScalarExpr::Column(key), - }) - .collect() -} - -impl Sizing { - /// How many approximate inputs `key`'s summary shares its query's - /// budget with. - fn share(&self, key: MeasureKey) -> usize { - self.shares.get(&key).copied().unwrap_or(1) - } -} - -/// What a measure's state reads from each row of `input`, resolvable there. -/// A SQL `count` reads [`counted_column`]; any other measure, a PromQL count -/// included, reads its input column (the sample value when it names none). -fn measure_input( - intent: &AggIntent, - input: &OperatorNode, - filter: Option<&Predicate>, -) -> Result { - let schema = &input.schema; - let read = match intent { - AggIntent::Count { .. } if input.result_kind == OperatorResultKind::Relation => { - SummaryInputExpr::Column(counted_column(schema, filter)?) - } - // A top-k ranks its child's aggregate output, not a column of its own. - AggIntent::TopK { .. } => SummaryInputExpr::Column( - crate::pass1::realization::ranked_value(input) - .map_err(|_| { - PerComputationCandidateError::Unsupported("Top-K has no ranked column") - })? - .1, - ), - _ => crate::pass1::realization::summarised_input(intent, schema).ok_or( - PerComputationCandidateError::Unsupported("input column outside child schema"), - )?, - }; - if let SummaryInputExpr::Column(column) = &read { - resolve_column_ref(column, schema).map_err(|_| { - PerComputationCandidateError::Unsupported("summary input column does not resolve") - })?; - } - Ok(read) -} - -/// The column a SQL `count` state updates on. The runtime skips NULL updates, -/// so it is a column non-null on every counted row: one the measure filter -/// proves non-null (`count(col)` lowers to the filter `col IS NOT NULL`), else -/// a non-nullable one, Float64 first (the runtime's numeric update). -fn counted_column( - schema: &Schema, - filter: Option<&Predicate>, -) -> Result { - fn not_null(expr: &ScalarExpr) -> Vec { - match expr { - ScalarExpr::IsNotNull(inner) => match **inner { - ScalarExpr::Column(column) => vec![column], - _ => vec![], - }, - ScalarExpr::BoolAnd(conjuncts) => conjuncts.iter().flat_map(not_null).collect(), - _ => vec![], - } - } - let non_nullable = |float: bool| { - schema - .fields - .iter() - .enumerate() - .filter_map(move |(id, field)| { - (!field.nullable - && (!float || field.dtype == FieldDataType::Plain(DataType::Float64))) - .then_some(id) - }) - }; - let reference = |id: usize| { - schema - .fields - .get(id) - .map(asap_types::ir::schema::Field::column_ref) - }; - filter - .map_or_else(Vec::new, |filter| not_null(&filter.0)) - .into_iter() - .chain(non_nullable(true)) - .chain(non_nullable(false)) - .find_map(|id| { - reference(id).filter(|column| resolve_column_ref(column, schema).ok() == Some(id)) - }) - .ok_or(PerComputationCandidateError::Unsupported( - "SQL count over rows with no column known to be non-null", - )) -} - -/// What each input row contributes, given what the measure reads -/// ([`measure_input`]), following the legacy realization rules: heap sketches -/// rank series identities, frequency sketches count values, and every other -/// family reads the measure's input. -fn summary_update( - intent: &AggIntent, - family: &FieldDataType, - reduction: &Reduction, - child: &Schema, - weight: SummaryInputExpr, -) -> Result { - let algorithm = match family { - FieldDataType::Sketch(kind, _) => Some(kind.algorithm()), - _ => None, - }; - Ok(match (intent, algorithm) { - (AggIntent::TopK { .. }, Some(_)) => { - if reduction.group_keys().is_some_and(|keys| keys.is_without()) { - return Err(PerComputationCandidateError::Unsupported( - "Top-K partitions given by `without`", - )); - } - let groups: Vec = reduction - .group_keys() - .into_iter() - .flat_map(|keys| keys.iter().copied()) - .collect(); - let item = if child.has_promql_series_identity() { - asap_types::ir::schema::topk::series_identity_item(child, &groups) - } else if child.closed { - // A closed child (an aggregate's output, say) lists every - // column: the item is its columns outside the partition and - // the ranked value, as the legacy current-series and counter - // rules rank them. - let SummaryInputExpr::Column(ranked) = &weight else { - return Err(PerComputationCandidateError::Unsupported( - "Top-K ranked value is not a column", - )); - }; - let value = resolve_column_ref(ranked, child).map_err(|_| { - PerComputationCandidateError::Unsupported("Top-K ranked value does not resolve") - })?; - let items: Vec<_> = child - .fields - .iter() - .enumerate() - .filter(|(index, _)| *index != value && !groups.contains(index)) - .map(|(_, field)| SummaryInputExpr::Column(field.column_ref())) - .collect(); - if items.is_empty() { - return Err(PerComputationCandidateError::Unsupported( - "Top-K item has no label outside its partition", - )); - } - SummaryInputExpr::Tuple(items) - } else { - let excluding = groups - .iter() - .filter_map(|&index| child.fields.get(index)) - .map(asap_types::ir::schema::Field::column_ref) - .collect(); - SummaryInputExpr::EntityIdentity(EntityIdentity::PromqlLabelSet { excluding }) - }; - SummaryUpdate { - item: Some(item), - // The ranked value ([`measure_input`]). - weight, - // Not proven non-negative; selection decides whether CMS is admissible. - weight_domain: WeightDomain::UnknownOrSigned, - } - } - (_, Some(SketchAlgorithm::UnivMon)) - | (AggIntent::Count { .. }, Some(SketchAlgorithm::Cms)) => SummaryUpdate { - item: Some(weight), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }, - _ => SummaryUpdate { - item: None, - weight, - weight_domain: WeightDomain::UnknownOrSigned, - }, - }) -} - -#[cfg(test)] -mod tests { - /// Aggregate replacement qualifiers survive both renaming and later input rebinding. - #[test] - fn aggregate_replacement_preserves_sql_field_identity() { - let original = - crate::test_support::lower_promql("sum by (job) (m)", AccuracyTarget::exact()); - let mut target = original.as_ref().clone(); - target.schema.fields[0].table = Some("metrics".into()); - let projected = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: target - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: original, - })) - .unwrap(); - assert_eq!(projected.schema.fields[0].table, None); - let replaced = with_names_of(&target, projected).unwrap(); - assert_eq!(replaced.schema, target.schema); - let rebound = replaced.with_new_children(Rc::clone).unwrap(); - assert_eq!(rebound.schema, target.schema); - } - - use super::*; - use asap_accuracy::composition::{StreamNorm, PRODUCT_OPERAND_DOMAINS}; - use asap_accuracy::propagation::operand_guarantee; - use asap_accuracy::test_support::{ - composed_guarantee, estimate_bound_over_raw_data_samples, estimate_has_bound, - estimates_raw_data_samples_of, fold_stats, local_guarantee, - raw_data_samples_sign_statistic, scalar_guarantee, sign_statistic, - source_distinct_guarantee, stream_norm, QUOTIENT_OPERAND_DOMAINS, - }; - use asap_types::ir::properties::guarantee::{ - AccuracyGuarantee, AccuracyMetric, BoundExpr, ProbabilityExpr, - }; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::schema::{SketchParams, SketchStatistic}; - /// Approximate count keeps one exact option, its accumulator, beside the - /// universal summary; plain exact execution is not offered. - #[test] - fn approximate_count_keeps_one_exact_accumulator_and_universal_choices() { - let choices = computation_candidates_for_intent(&AggIntent::Count { - accuracy: AccuracyTarget::max_error(0.05).failure_probability(0.01), - }) - .unwrap() - .candidates; - assert!(!choices.contains(&HowToComputeAggregateMeasure::Exact)); - assert_eq!( - choices[0], - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count]) - ); - assert!(choices.contains(&HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::UnivMon - ))); - } - - /// Count Sketch's signed counters give no total, so Pass 1 offers no - /// Count Sketch for a count, and every count candidate it offers - /// answers the total. - #[test] - fn a_count_offers_no_count_sketch() { - let choices = computation_candidates_for_intent(&AggIntent::Count { - accuracy: AccuracyTarget::max_error(0.05).failure_probability(0.01), - }) - .unwrap() - .candidates; - assert!(!choices.contains(&HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::CountSketch - ))); - let total = sketch_statistic(&AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }) - .unwrap(); - for choice in &choices { - if let HowToComputeAggregateMeasure::Summary(algorithm) = choice { - assert!( - asap_types::ir::schema::FamilyCapability::of_per_instance_sketch(algorithm) - .answers(&total), - "{algorithm:?}" - ); - } - } - } - - /// Guarantee composition through scalar and vector arithmetic (#669). - mod guarantee_composition { - use super::*; - use crate::test_support::lower_promql; - use asap_types::ir::properties::AccuracyTarget; - - fn with_bound(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - metric, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - fn abs(bound: f64, delta: f64) -> AccuracyGuarantee { - with_bound(AccuracyMetric::ABSOLUTE, bound, delta) - } - fn rel(bound: f64) -> AccuracyGuarantee { - with_bound(AccuracyMetric::Relative, bound, 0.0) - } - /// `sum(m)` carrying `guarantee`. - fn node(guarantee: AccuracyGuarantee) -> Rc { - node_over("sum(m)", guarantee) - } - /// `query` carrying `guarantee`. - fn node_over(query: &str, guarantee: AccuracyGuarantee) -> Rc { - let node = lower_promql(query, AccuracyTarget::exact()); - Rc::new((*node).clone().with_guarantee(Some(guarantee))) - } - fn scalar(guarantee: AccuracyGuarantee) -> ScalarExpr { - ScalarExpr::PromqlScalarFromVector(node(guarantee)) - } - fn lit(value: f64) -> ScalarExpr { - ScalarExpr::literal_f64(value) - } - fn arith(op: ArithmeticOpKind, left: ScalarExpr, right: ScalarExpr) -> ScalarExpr { - ScalarExpr::Arithmetic { - op, - left: Box::new(left), - right: Box::new(right), - semantics: ExprSemantics::Promql, - } - } - fn call(name: &str, args: Vec) -> ScalarExpr { - ScalarExpr::FunctionCall { - name: name.into(), - args, - } - } - fn of(expr: &ScalarExpr) -> Option { - scalar_guarantee(expr, &|_| None) - } - fn bound(g: &AccuracyGuarantee) -> f64 { - g.spec.error.evaluate().expect("a resolved bound") - } - fn delta(g: &AccuracyGuarantee) -> f64 { - g.spec - .failure_probability - .evaluate() - .expect("a resolved probability") - } - fn close(a: f64, b: f64) -> bool { - (a - b).abs() < 1e-12 - } - - /// Absolute errors add through `x ± y`, with the union bound on the - /// failure probability; `−x` and `x ± c` keep the error, `c · x` and - /// `x / c` scale it by `|c|` and `1/|c|`, division by a literal zero - /// has none. - #[test] - fn absolute_errors_compose_through_arithmetic() { - use ArithmeticOpKind::*; - let (x, y) = (scalar(abs(0.01, 0.001)), scalar(abs(0.02, 0.002))); - for op in [Add, Sub] { - let g = of(&arith(op.clone(), x.clone(), y.clone())).unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE, "{op:?}"); - assert!(close(bound(&g), 0.03) && close(delta(&g), 0.003), "{op:?}"); - for shifted in [ - arith(op.clone(), x.clone(), lit(5.0)), - arith(op.clone(), lit(5.0), x.clone()), - arith(op.clone(), x.clone(), ScalarExpr::EvalTimestamp), - ] { - let g = of(&shifted).unwrap(); - assert!(close(bound(&g), 0.01) && close(delta(&g), 0.001), "{op:?}"); - } - } - let negated = ScalarExpr::Negative { - expr: Box::new(x.clone()), - semantics: ExprSemantics::Promql, - }; - assert!(close(bound(&of(&negated).unwrap()), 0.01)); - for (expr, expected) in [ - (arith(Mul, lit(-3.0), x.clone()), 0.03), - (arith(Mul, x.clone(), lit(3.0)), 0.03), - (arith(Div, x.clone(), lit(-4.0)), 0.0025), - ] { - let g = of(&expr).unwrap(); - assert!( - close(bound(&g), expected) && close(delta(&g), 0.001), - "{expr:?}" - ); - } - for none in [ - arith(Div, x.clone(), lit(0.0)), - arith(Mul, x.clone(), ScalarExpr::EvalTimestamp), - arith(Pow, x.clone(), lit(2.0)), - ] { - assert_eq!(of(&none), None, "{none:?}"); - } - } - - /// Count-Min and Count Sketch errors are additive: through - /// arithmetic they compose as absolute errors, each normalized bound - /// times the (symbolic) stream norm of its summary's source, here - /// over each source's raw value column, for plan selection to bind. - #[test] - fn normalized_additive_errors_compose_as_absolute_errors() { - let cms = with_bound(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01, 0.001); - let count_sketch = with_bound(AccuracyMetric::ABSOLUTE_STREAM_L2, 0.02, 0.002); - // Summaries of each source's raw value column. - let raw = |query: &str, guarantee: AccuracyGuarantee| { - ScalarExpr::PromqlScalarFromVector(crate::test_support::estimate_over( - crate::test_support::weighted( - crate::test_support::cms_summary(query), - SummaryInputExpr::Column(ColumnRef::SampleValue), - ), - guarantee, - )) - }; - let g = of(&arith( - ArithmeticOpKind::Sub, - raw("count by (job) (m)", cms.clone()), - raw("count by (job) (n)", count_sketch), - )) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert!(close(delta(&g), 0.003)); - let norms = |statistic: &str| match statistic { - "stream_l1_norm[m]" => Some(100.0), - "stream_l2_norm[n]" => Some(10.0), - _ => None, - }; - assert!(g.spec.error.has_unknown()); - assert!(close(g.bind(&norms).spec.error.evaluate().unwrap(), 1.2)); - let doubled = of(&arith( - ArithmeticOpKind::Mul, - lit(2.0), - raw("count by (job) (m)", cms), - )) - .unwrap(); - assert!(close( - doubled.bind(&norms).spec.error.evaluate().unwrap(), - 2.0 - )); - } - - /// A product of two estimates has an absolute bound - /// `|x|·e_y + |y|·e_x + e_x·e_y`, and a quotient with an approximate - /// denominator one that needs the denominator bounded away from - /// zero: both need operand domains, so Stage 1 leaves them symbolic - /// for plan selection to bind. Relative errors need no domain: a - /// product has the closed form `(1 + a)(1 + b) − 1`, a quotient - /// `(a + b) / (1 − b)` (`a = 0` for an exact numerator). The failure - /// probability is the union bound. A rank error composes to none. - #[test] - fn products_and_quotients_of_estimates_have_closed_or_symbolic_bounds() { - use ArithmeticOpKind::*; - let unknown = |statistic: &str| BoundExpr::Unknown { - statistic: statistic.into(), - }; - let (x, y) = (scalar(abs(0.01, 0.001)), scalar(abs(0.02, 0.002))); - let product = of(&arith(Mul, x.clone(), y.clone())).unwrap(); - assert_eq!(product.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!(product.spec.error, unknown(PRODUCT_OPERAND_DOMAINS)); - assert!(close(delta(&product), 0.003)); - for quotient in [ - arith(Div, x.clone(), y.clone()), - arith(Div, lit(1.0), x.clone()), - arith(Div, x.clone(), scalar(rel(0.01))), - ] { - let g = of("ient).unwrap(); - assert_eq!( - g.spec.accuracy_metric, - AccuracyMetric::ABSOLUTE, - "{quotient:?}" - ); - assert_eq!( - g.spec.error, - unknown(QUOTIENT_OPERAND_DOMAINS), - "{quotient:?}" - ); - } - let ratio = of(&arith( - Div, - scalar(with_bound(AccuracyMetric::Relative, 0.01, 0.001)), - scalar(with_bound(AccuracyMetric::Relative, 0.02, 0.002)), - )) - .unwrap(); - assert_eq!(ratio.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(close(bound(&ratio), (0.01 + 0.02) / (1.0 - 0.02))); - assert!(close(delta(&ratio), 0.003)); - let reciprocal = of(&arith(Div, lit(2.0), scalar(rel(0.02)))).unwrap(); - assert_eq!(reciprocal.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(close(bound(&reciprocal), 0.02 / (1.0 - 0.02))); - // A denominator bound of 1 or more leaves the ratio unbounded. - assert_eq!(of(&arith(Div, scalar(rel(0.01)), scalar(rel(1.0)))), None); - // A relative bound still symbolic in Stage 1 has no ratio rule. - let symbolic = AccuracyGuarantee::new( - AccuracyMetric::Relative, - unknown("values_sign"), - ProbabilityExpr::Zero, - vec![], - ); - assert_eq!(of(&arith(Div, scalar(symbolic), scalar(rel(0.01)))), None); - let relative = of(&arith(Mul, scalar(rel(0.01)), scalar(rel(0.02)))).unwrap(); - assert_eq!(relative.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(close(bound(&relative), 0.01 + 0.02 + 0.0002)); - let rank = scalar(with_bound(AccuracyMetric::Rank, 0.01, 0.001)); - assert_eq!(of(&arith(Mul, rank.clone(), x.clone())), None); - assert_eq!(of(&arith(Div, x, rank)), None); - } - - /// A relative error survives negation and scaling by an exact value, - /// but not a shift (`x + c`) or a sum or difference of estimates - /// (cancellation); a rank error composes to no guarantee (a rank - /// interval counts as none in the MVP). - #[test] - fn relative_and_rank_errors_compose_only_where_sound() { - use ArithmeticOpKind::*; - let x = scalar(rel(0.01)); - for kept in [ - ScalarExpr::Negative { - expr: Box::new(x.clone()), - semantics: ExprSemantics::Promql, - }, - arith(Mul, x.clone(), lit(-2.0)), - arith(Div, x.clone(), ScalarExpr::EvalTimestamp), - ] { - let g = of(&kept).unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative, "{kept:?}"); - assert!(close(bound(&g), 0.01), "{kept:?}"); - } - for none in [ - arith(Add, x.clone(), lit(1.0)), - arith(Sub, x.clone(), scalar(rel(0.01))), - ] { - assert_eq!(of(&none), None, "{none:?}"); - } - let rank = scalar(with_bound(AccuracyMetric::Rank, 0.01, 0.001)); - for none in [ - arith(Sub, rank.clone(), rank.clone()), - arith(Mul, rank.clone(), lit(2.0)), - call("promql_clamp_max", vec![rank, lit(1.0)]), - ] { - assert_eq!(of(&none), None, "{none:?}"); - } - } - - /// `clamp`, `clamp_min`, `clamp_max` and `abs` are 1-Lipschitz and - /// monotone in each argument: the result keeps the largest absolute - /// (or relative) error of its arguments, with the union bound on the - /// failure probability. - #[test] - fn clamp_and_abs_keep_the_largest_argument_error() { - let x = scalar(abs(0.01, 0.001)); - for expr in [ - call("promql_clamp", vec![x.clone(), lit(0.0), lit(10.0)]), - call("promql_clamp_min", vec![x.clone(), lit(0.0)]), - call("promql_clamp_max", vec![x.clone(), lit(10.0)]), - call("promql_abs", vec![x.clone()]), - call("abs", vec![x.clone()]), - ] { - let g = of(&expr).unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE, "{expr:?}"); - assert!( - close(bound(&g), 0.01) && close(delta(&g), 0.001), - "{expr:?}" - ); - } - let bounded = of(&call( - "promql_clamp", - vec![x.clone(), scalar(abs(0.03, 0.002)), lit(10.0)], - )) - .unwrap(); - assert!(close(bound(&bounded), 0.03) && close(delta(&bounded), 0.003)); - let relative = of(&call( - "promql_clamp", - vec![scalar(rel(0.01)), scalar(rel(0.02)), lit(10.0)], - )) - .unwrap(); - assert_eq!(relative.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(close(bound(&relative), 0.02)); - let mixed = call("promql_clamp_min", vec![x, scalar(rel(0.02))]); - assert_eq!(of(&mixed), None); - assert_eq!(of(&call("promql_floor", vec![scalar(rel(0.01))])), None); - } - - /// A comparison over an estimate has no guarantee on its output (a - /// band guarantee is later work); over exact operands it is exact. - #[test] - fn comparisons_over_estimates_have_no_guarantee() { - let compare = |left: ScalarExpr| ScalarExpr::Compare { - left: Box::new(left), - op: CompareOpKind::Gt, - right: Box::new(lit(5.0)), - semantics: ExprSemantics::Promql, - }; - assert_eq!(of(&compare(scalar(abs(0.01, 0.001)))), None); - assert!(of(&compare(scalar(AccuracyGuarantee::exact("exact")))) - .is_some_and(|g| g.is_exact())); - } - - /// A SQL scalar subquery inherits its relation's guarantee, as - /// PromQL `scalar(v)` does. - #[test] - fn a_scalar_subquery_inherits_its_guarantee() { - let g = abs(0.01, 0.001); - let expr = arith( - ArithmeticOpKind::Add, - ScalarExpr::ScalarSubquery(node(g.clone())), - lit(1.0), - ); - assert!(close(bound(&of(&expr).unwrap()), 0.01)); - } - - /// Two estimates count their failure probabilities once each (union - /// bound), even when they read one shared summary: the built-in - /// guarantees hold per estimate, not simultaneously. The same - /// planned estimate read twice fails or holds as one event, so its - /// failure probability counts once. - #[test] - fn failure_probability_counts_once_per_distinct_estimate() { - let shared = node(abs(0.01, 0.001)); - // Equal to `shared`, but planned separately. - let other = node(abs(0.01, 0.001)); - let same = ScalarExpr::PromqlScalarFromVector(shared.clone()); - let twice = of(&arith(ArithmeticOpKind::Add, same.clone(), same.clone())).unwrap(); - assert!(close(bound(&twice), 0.02) && close(delta(&twice), 0.001)); - let distinct = of(&arith( - ArithmeticOpKind::Add, - same, - ScalarExpr::PromqlScalarFromVector(other.clone()), - )) - .unwrap(); - assert!(close(bound(&distinct), 0.02) && close(delta(&distinct), 0.002)); - let vector = |lhs: &Rc, rhs: &Rc| { - let node = OperatorNode::new(Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Sub), - vector_match: None, - }, - return_bool: false, - lhs: lhs.clone(), - rhs: rhs.clone(), - })) - .unwrap(); - composed_guarantee(&node).unwrap() - }; - assert!(close(delta(&vector(&shared, &shared)), 0.001)); - assert!(close(delta(&vector(&shared, &other)), 0.002)); - } - - /// `left JOIN right ON pred`, composed. - fn join( - kind: JoinKind, - left: &Rc, - right: &Rc, - pred: ScalarExpr, - ) -> OperatorNode { - let mut node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Join { - kind, - pred: Predicate(pred), - left: left.clone(), - right: right.clone(), - })) - .unwrap(); - node.guarantee = composed_guarantee(&node); - node - } - fn column(id: usize) -> ScalarExpr { - ScalarExpr::Column(id) - } - fn equals(left: ScalarExpr, right: ScalarExpr) -> ScalarExpr { - ScalarExpr::Compare { - left: Box::new(left), - op: CompareOpKind::Eq, - right: Box::new(right), - semantics: ExprSemantics::Sql, - } - } - /// `CASE WHEN condition THEN NULL ELSE otherwise END` over `child`, - /// composed. - fn case_over( - child: &OperatorNode, - condition: ScalarExpr, - otherwise: ScalarExpr, - ) -> Option { - let node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { - cols: vec![ProjectItem { - alias: Some("v".into()), - expr: ScalarExpr::Case { - operand: None, - branches: vec![(condition, ScalarExpr::Literal(ScalarValue::Null))], - else_expr: Some(Box::new(otherwise)), - }, - }], - qualifier: None, - child: Rc::new(child.clone()), - })) - .unwrap(); - composed_guarantee(&node) - } - /// An approximate node and an exact one, and the approximate one's - /// column count (its columns come first in a join). - fn approximate_and_exact() -> (Rc, Rc, usize) { - let approximate = node(rel(0.01)); - let exact = node_over("count(m)", AccuracyGuarantee::exact("test")); - let width = approximate.schema.fields.len(); - (approximate, exact, width) - } - - /// A join whose other input is exact and whose predicate reads only - /// exact columns keeps the approximate input's guarantee: a cross - /// join, or a key equality on the exact input's columns. - #[test] - fn join_beside_an_exact_input_on_exact_columns_keeps_the_guarantee() { - let (approximate, exact, width) = approximate_and_exact(); - for (kind, pred) in [ - ( - JoinKind::Cross, - ScalarExpr::Literal(ScalarValue::Boolean(true)), - ), - (JoinKind::Inner, equals(column(width), lit(0.0))), - (JoinKind::Left, equals(column(width), lit(0.0))), - ] { - assert_eq!( - join(kind.clone(), &approximate, &exact, pred).guarantee, - Some(rel(0.01)), - "{kind:?}" - ); - } - // Either side. - assert_eq!( - join( - JoinKind::Cross, - &exact, - &approximate, - ScalarExpr::Literal(ScalarValue::Boolean(true)) - ) - .guarantee, - Some(rel(0.01)) - ); - } - - /// A join keyed on an approximate input's column, a semi-join, or - /// two approximate inputs have no guarantee (column-level tracking - /// is #689). - #[test] - fn join_on_an_approximate_key_has_no_guarantee() { - let (approximate, exact, width) = approximate_and_exact(); - assert_eq!( - join( - JoinKind::Inner, - &approximate, - &exact, - equals(column(0), column(width)) - ) - .guarantee, - None - ); - assert_eq!( - join( - JoinKind::Semi, - &exact, - &approximate, - ScalarExpr::Literal(ScalarValue::Boolean(true)) - ) - .guarantee, - None - ); - assert_eq!( - join( - JoinKind::Cross, - &approximate, - &approximate, - ScalarExpr::Literal(ScalarValue::Boolean(true)) - ) - .guarantee, - None - ); - } - - /// A `CASE` whose condition reads only exact columns keeps each - /// branch's guarantee (a NULL branch is exact); one testing an - /// approximate column has none. - #[test] - fn case_on_exact_columns_keeps_the_branch_guarantee() { - let (approximate, exact, width) = approximate_and_exact(); - let joined = join( - JoinKind::Cross, - &approximate, - &exact, - ScalarExpr::Literal(ScalarValue::Boolean(true)), - ); - let value = approximate.schema.fields.len() - 1; - assert_eq!( - case_over(&joined, equals(column(width), lit(0.0)), column(value)), - Some(rel(0.01)) - ); - assert_eq!( - case_over(&joined, equals(column(value), lit(0.0)), column(value)), - None - ); - } - - /// `node` projecting `cols` over `child`, composed. - fn project(child: &OperatorNode, cols: Vec) -> OperatorNode { - let mut node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { - cols: cols - .into_iter() - .enumerate() - .map(|(i, expr)| ProjectItem { - alias: Some(format!("c{i}")), - expr, - }) - .collect(), - qualifier: None, - child: Rc::new(child.clone()), - })) - .unwrap(); - node.guarantee = composed_guarantee(&node); - node - } - - /// Over a join, a projection reads each column's own guarantee, - /// not the child's sample value (which may be an exact input's - /// column): `sum(m) * 1000` beside the exact side's `value` has - /// the scaled error 1000, not the approximate side's 1. A - /// projection over that computed projection has no per-column - /// guarantees, so it has none (#689). - #[test] - fn projection_over_a_join_reads_each_columns_guarantee() { - let approximate = node(abs(1.0, 0.01)); - let exact = node_over("m", AccuracyGuarantee::exact("test")); - let joined = join( - JoinKind::Cross, - &approximate, - &exact, - ScalarExpr::Literal(ScalarValue::Boolean(true)), - ); - assert_eq!(joined.guarantee, Some(abs(1.0, 0.01))); - let sum = approximate.schema.fields.len() - 1; - let value = approximate.schema.fields.len() + exact.schema.fields.len() - 1; - let scaled = project( - &joined, - vec![ - arith(ArithmeticOpKind::Mul, column(sum), lit(1000.0)), - column(value), - ], - ); - assert_eq!(scaled.guarantee.as_ref().map(bound), Some(1000.0)); - let stacked = project( - &scaled, - vec![arith(ArithmeticOpKind::Add, column(1), lit(1.0)), column(0)], - ); - assert_eq!(stacked.guarantee, None); - } - } - - /// Which stream norm bounds a Count-Min or Count Sketch estimate (#669). - mod stream_norms { - use super::*; - use crate::test_support::{cms_summary, weighted}; - use asap_accuracy::composition::STREAM_L1_NORM; - use asap_accuracy::test_support::{source_norm_statistic, STREAM_ROW_COUNT}; - - /// A count (constant weight) is bounded by its weight times the - /// source's row count; a sum of the source's raw value column by - /// the source's norm; a sum of a computed value (`m * 1000`, - /// `rate(…)`) by nothing that can be bound. - #[test] - fn a_norm_follows_the_update_weight() { - let count = Rc::new(cms_summary("count by (job) (m)")); - assert_eq!( - stream_norm(&count), - Some(StreamNorm::ConstantWeight { - weight: 1.0, - source: "m".into() - }) - ); - let raw = weighted( - cms_summary("count by (job) (m)"), - SummaryInputExpr::Column(ColumnRef::SampleValue), - ); - assert_eq!(stream_norm(&raw), Some(StreamNorm::Source("m".into()))); - for computed in ["count by (job) (m * 1000)", "count by (job) (rate(m[1m]))"] { - let summary = weighted( - cms_summary(computed), - SummaryInputExpr::Column(ColumnRef::SampleValue), - ); - assert_eq!(stream_norm(&summary), None, "{computed}"); - } - } - - /// The absolute form of a Count-Min bound names the statistic its - /// update weight allows: the source's norm for its raw value - /// column, the weight times the source's row count for a count. - #[test] - fn the_absolute_bound_names_the_weighted_norm() { - let frequency = AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE_STREAM_L1, - BoundExpr::Constant { value: 0.01 }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - let estimate = |summary: Rc| { - operand_guarantee(&crate::test_support::estimate_over( - summary, - frequency.clone(), - )) - .unwrap() - }; - let raw = estimate(weighted( - cms_summary("count by (job) (m)"), - SummaryInputExpr::Column(ColumnRef::SampleValue), - )); - let l1 = source_norm_statistic(STREAM_L1_NORM, "m"); - assert_eq!( - raw.bind(&|s| (s == l1).then_some(100.0)) - .spec - .error - .evaluate(), - Some(1.0) - ); - let count = estimate(Rc::new(cms_summary("count by (job) (m)"))); - assert!(count - .bind(&|s| (s == l1).then_some(100.0)) - .spec - .error - .has_unknown()); - let rows = source_norm_statistic(STREAM_ROW_COUNT, "m"); - assert_eq!( - count - .bind(&|s| (s == rows).then_some(100.0)) - .spec - .error - .evaluate(), - Some(1.0) - ); - } - } - - fn inventory(query: &str) -> PerComputationCandidates { - let root = crate::test_support::lower_promql( - query, - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - // Per-series state ranks each series' identity, which the rows carry - // as a column; a query over aggregated rows is planned over labels. - let root = asap_types::ir::schema::schema_support::with_promql_series_identity(&root) - .unwrap_or(root); - generate_per_computation_candidates(&crate::stage0::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ])) - .unwrap() - } - - /// Every candidate of target `t`, then of its fused computation, composed - /// with every other target at its first candidate, with the candidate. - fn composed_with( - inventory: &PerComputationCandidates, - t: usize, - ) -> Vec<(HowToComputeAggregateMeasure, Rc)> { - (0..inventory.targets[t].choice_count()) - .map(|c| { - let mut choice = vec![0; inventory.targets.len()]; - choice[t] = c; - let composed = compose_logical_candidate(inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - let (candidate, _) = inventory.targets[t].computation_for(c).unwrap(); - (candidate.clone(), root.clone()) - }) - .collect() - } - - fn heap_states(root: &Rc) -> Vec> { - OperatorNode::reachable(root) - .into_iter() - .filter(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if kind.algorithm().capability().answers(StatisticKind::TopK)) - }) - .collect() - } - - fn topk_target(inventory: &PerComputationCandidates) -> usize { - inventory - .targets - .iter() - .position(|t| { - matches!( - inventory.aggregate(t).non_asap(), - Some(NonASAPOp::Aggregate { measures, .. }) - if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) - ) - }) - .expect("a top-k target") - } - - /// Example 1's Q2: the top-k over a per-series `sum_over_time` keeps - /// only its exact candidate and gets a fused whole-expression - /// computation with a Count-Min and a CountSketch heap, which replace the - /// heaps over the exact sum. The fused computation absorbs the inner sum: composed, no - /// aggregate is computed and the heap reads the range selector's raw - /// samples. - #[test] - fn whole_expression_topk_replaces_the_heap_over_the_exact_sum() { - let inventory = inventory("topk by (job) (10, sum_over_time(m[1m]))"); - let topk = topk_target(&inventory); - let target = &inventory.targets[topk]; - assert_eq!( - target.ways_to_compute, - [HowToComputeAggregateMeasure::Exact] - ); - let fused = target - .fused_computation - .as_ref() - .expect("a fused computation"); - assert_eq!( - fused.ways_to_compute, - [ - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - ] - ); - let inner = fused.absorbed_targets[0]; - assert_eq!(inventory.targets[inner].ways_to_compute.len(), 1); - assert_eq!(inventory.combinations(), 3); - for (candidate, root) in composed_with(&inventory, topk) { - if candidate == HowToComputeAggregateMeasure::Exact { - continue; - } - assert!( - !OperatorNode::reachable(&root).iter().any(|n| matches!( - n.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) || matches!( - n.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - )), - "{candidate:?}: the inner sum is not computed" - ); - let [state] = &heap_states(&root)[..] else { - panic!("one heap") - }; - let Some(ASAPOp::SummaryAgg { child, .. }) = state.asap() else { - unreachable!() - }; - assert!(matches!( - child.non_asap(), - Some(NonASAPOp::TimeRange { .. }) - )); - } - } - - /// The whole-expression heap reads the inner sum's input, so the outer - /// top-k's partition keys (indices into the sum's output) are moved onto - /// that input by column. - #[test] - fn whole_expression_topk_partitions_index_the_heap_input() { - let root = crate::test_support::lower_promql( - "topk by (job) (2, sum by (service, job) (rate(m[1m])))", - AccuracyTarget::max_error(0.1), - ); - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&[QueryRoot::Operator(root)]), - ) - .unwrap(); - let topk = topk_target(&inventory); - let mut heaps = 0; - for (candidate, root) in composed_with(&inventory, topk) { - // The top-k's own candidate is exact; its summaries are the - // fused computation's. - if !matches!(candidate, HowToComputeAggregateMeasure::Summary(_)) { - continue; - } - for state in heap_states(&root) { - let Some(ASAPOp::SummaryAgg { - child, reduction, .. - }) = state.asap() - else { - unreachable!() - }; - let names: Vec<_> = reduction - .group_keys() - .expect("partitioned by job") - .keys() - .iter() - .map(|&index| child.schema.fields[index].name.as_str()) - .collect(); - assert_eq!(names, ["job"]); - heaps += 1; - } - } - assert_eq!( - heaps, 2, - "a whole-expression Count-Min and CountSketch heap" - ); - } - - /// How many whole-workload choices enumeration keeps: every digit over - /// its target's `choice_count`, less those [`absorbed_choice_varies`] - /// skips. - /// - /// [`absorbed_choice_varies`]: PerComputationCandidates::absorbed_choice_varies - fn enumerated_choices(inventory: &PerComputationCandidates) -> usize { - let mut enumerated = 0; - let mut choice = vec![0; inventory.targets.len()]; - loop { - if !inventory.absorbed_choice_varies(&choice) { - enumerated += 1; - } - let mut carry = true; - for (digit, target) in choice.iter_mut().zip(&inventory.targets).rev() { - *digit += 1; - if *digit < target.choice_count() { - carry = false; - break; - } - *digit = 0; - } - if carry { - return enumerated; - } - } - } - - /// `topk(k, sum by (G) (sum_over_time(m[W])))` and its `count_over_time` - /// form rank the groups `G` by the sum (count) of their raw data samples: the - /// top-k keeps its exact candidate and gets the fused whole-expression - /// Count-Min and CountSketch heaps, which absorb both the grouped sum and - /// the per-series sum (count) beneath it. Composed, no aggregate is - /// computed: the heap reads the range selector's raw data samples, keyed by - /// `G` (less the top-k's own `by` labels, its partitions), weighted by - /// the sample value (a sum, an open requirement on `m`) or by one (a - /// count, proven non-negative). - #[test] - fn whole_expression_topk_over_a_grouped_sum_of_range_sums_reads_the_raw_data_samples() { - let samples = WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }; - let unit = WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }; - let value = SummaryInputExpr::Column(ColumnRef::SampleValue); - let one = SummaryInputExpr::Constant(1.0); - for (query, weight, domain, partitions) in [ - ( - "topk(3, sum by (region) (sum_over_time(m[1m])))", - &value, - &samples, - &[][..], - ), - ( - "topk(3, sum by (region) (count_over_time(m[1m])))", - &one, - &unit, - &[], - ), - ( - "topk by (job) (3, sum by (job, region) (sum_over_time(m[1m])))", - &value, - &samples, - &["job"], - ), - ] { - let inventory = inventory(query); - let topk = topk_target(&inventory); - let target = &inventory.targets[topk]; - assert_eq!( - target.ways_to_compute, - [HowToComputeAggregateMeasure::Exact], - "{query}" - ); - let fused = target - .fused_computation - .as_ref() - .expect("a fused computation"); - assert_eq!( - fused.ways_to_compute, - [ - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - ], - "{query}" - ); - assert_eq!( - fused.absorbed_targets.len(), - 2, - "{query}: both aggregates beneath" - ); - assert_eq!( - inventory.combinations(), - enumerated_choices(&inventory), - "{query}" - ); - let mut heaps = 0; - for (candidate, root) in composed_with(&inventory, topk) { - if candidate == HowToComputeAggregateMeasure::Exact { - continue; - } - assert!( - !OperatorNode::reachable(&root).iter().any(|n| matches!( - n.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) || matches!( - n.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - )), - "{query} {candidate:?}: no sum is computed" - ); - let [state] = &heap_states(&root)[..] else { - panic!("{query}: one heap") - }; - let Some(ASAPOp::SummaryAgg { - child, - input, - reduction, - .. - }) = state.asap() - else { - unreachable!() - }; - assert!( - matches!(child.non_asap(), Some(NonASAPOp::TimeRange { .. })), - "{query}: the heap reads the raw data samples" - ); - assert_eq!( - input.item, - Some(SummaryInputExpr::Column(ColumnRef::Named("region".into()))), - "{query}" - ); - assert_eq!(&input.weight, weight, "{query}"); - assert_eq!(&input.weight_domain, domain, "{query}"); - let names: Vec<_> = reduction - .group_keys() - .map_or(&[][..], |keys| keys.keys()) - .iter() - .map(|&index| child.schema.fields[index].name.as_str()) - .collect(); - assert_eq!(names, partitions, "{query}"); - // The answer has the top-k's columns, `region` among them. - assert!( - root.schema.fields.iter().any(|f| f.name == "region"), - "{query}: {:?}", - root.schema - ); - heaps += 1; - } - assert_eq!(heaps, 2, "{query}"); - } - } - - /// The weight domain of every heap-sketch update in every candidate of - /// `query`'s top-k. - fn heap_weight_domains(query: &str) -> Vec { - let inventory = inventory(query); - let topk = topk_target(&inventory); - composed_with(&inventory, topk) - .iter() - .flat_map(|(_, root)| heap_states(root)) - .map(|state| match state.asap() { - Some(ASAPOp::SummaryAgg { input, .. }) => input.weight_domain.clone(), - _ => unreachable!(), - }) - .collect() - } - - /// Pass 1 does not read the metric's type: heap weights that are raw - /// samples of `m` carry the open requirement that `m` is a counter, under - /// any name (`m_total` included), for plan selection to confirm. - #[test] - fn heap_weights_over_raw_data_samples_record_an_open_requirement() { - for (query, metric) in [ - ("topk by (job) (10, sum_over_time(m[1m]))", "m"), - ("topk by (job) (10, sum_over_time(m_total[1m]))", "m_total"), - ] { - let domains = heap_weight_domains(query); - assert_eq!(domains.len(), 2, "{query}"); - assert!( - domains.iter().all(|d| *d - == WeightDomain::RawDataSamplesOfMetric { - metric: metric.into() - }), - "{query}: {domains:?}" - ); - } - } - - /// Only raw data samples and plain sums of them: arithmetic, `rate` and other - /// functions end the requirement. - #[test] - fn raw_data_samples_of_metric_ends_at_any_other_operator() { - for (query, metric) in [ - ("sum_over_time(m[1m])", Some("m")), - ("sum by (job) (sum_over_time(m[1m]))", Some("m")), - ("sum_over_time(m[1m]) - 100", None), - ("-sum_over_time(m[1m])", None), - ("rate(m[1m])", None), - ("max_over_time(m[1m])", None), - ] { - let root = crate::test_support::lower_promql(query, AccuracyTarget::exact()); - assert_eq!( - raw_data_samples_of_metric(&root).as_deref(), - metric, - "{query}" - ); - } - } - - /// An avg's exact states are its sum then its count: any other order - /// is refused rather than divided the wrong way round. - #[test] - fn avg_states_out_of_order_are_refused() { - let mut inventory = inventory("avg by (job) (m)"); - assert_eq!( - inventory.targets[0].ways_to_compute, - [HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum, - ExactKind::Count - ])] - ); - assert!(compose_logical_candidate(&inventory, &[0]).is_ok()); - inventory.targets[0].ways_to_compute = vec![ - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count, ExactKind::Sum]), - ]; - assert!(matches!( - compose_logical_candidate(&inventory, &[0]), - Err(PerComputationCandidateError::Unsupported(_)) - )); - } - - /// The target whose fused computation is a nested-sum merge. - fn nested_sum_target(inventory: &PerComputationCandidates) -> Option { - let merge = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); - inventory.targets.iter().position(|t| { - t.fused_computation - .as_ref() - .is_some_and(|fused| fused.ways_to_compute == [merge.clone()]) - }) - } - - /// `sum by (job) (sum_over_time(m[5m]))` gets a fused nested sum - /// beside its own accumulator: one exact Sum accumulator over the range's - /// raw data samples, grouped by `job`, which absorbs the inner per-series sum. - /// Composed, no per-series state is built, the value is exact and the - /// output columns are the outer sum's. - #[test] - fn nested_sum_merges_into_one_exact_accumulator() { - let inventory = inventory("sum by (job) (sum_over_time(m[5m]))"); - let outer = nested_sum_target(&inventory).expect("a nested-sum merge"); - let target = &inventory.targets[outer]; - let sum = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); - assert_eq!(target.ways_to_compute, std::slice::from_ref(&sum)); - let fused = target - .fused_computation - .as_ref() - .expect("a fused computation"); - assert_eq!(fused.ways_to_compute, [sum]); - let inner = fused.absorbed_targets[0]; - assert_eq!(inventory.targets[inner].ways_to_compute.len(), 1); - assert_eq!(inventory.combinations(), 2); - let composed = composed_with(&inventory, outer); - let (_, separate) = &composed[0]; - let (_, merged) = &composed[1]; - let states: Vec<_> = OperatorNode::reachable(merged) - .into_iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect(); - let [state] = states.as_slice() else { - panic!("one accumulator: {states:?}") - }; - let Some(ASAPOp::SummaryAgg { - child, - family, - reduction, - .. - }) = state.asap() - else { - unreachable!() - }; - assert_eq!( - *family, - FieldDataType::ExactAggregate(ExactKind::Sum, asap_types::ir::schema::ExactParams::Sum) - ); - assert!(matches!( - child.non_asap(), - Some(NonASAPOp::TimeRange { .. }) - )); - let Reduction::Reduce(keys) = reduction else { - panic!("grouped by job: {reduction:?}") - }; - assert_eq!( - keys.keys() - .iter() - .map(|&k| child.schema.fields[k].name.as_str()) - .collect::>(), - ["job"] - ); - assert!(merged - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact)); - let names = |node: &OperatorNode| { - node.schema - .fields - .iter() - .map(|f| f.name.clone()) - .collect::>() - }; - assert_eq!(names(merged), names(separate)); - merged.validate_structure().unwrap(); - } - - /// The merge needs a sum over a sum's value: an outer sum over a rate, - /// an outer count, or an outer sum over an inner `max` gets none. - #[test] - fn nested_sum_merge_needs_a_sum_of_sums() { - for query in [ - "sum by (job) (rate(m[5m]))", - "count by (job) (sum_over_time(m[5m]))", - "sum by (job) (max_over_time(m[5m]))", - ] { - assert_eq!(nested_sum_target(&inventory(query)), None, "{query}"); - } - } - - /// A per-series sum drops the metric name, but the raw data samples keep it: - /// grouping the merge by `__name__` would answer the metric's name where - /// the nested sums answer none, so no merge is offered. - #[test] - fn nested_sum_merge_never_groups_by_the_dropped_metric_name() { - let query = "sum by (__name__) (sum_over_time(m[5m]))"; - assert_eq!(nested_sum_target(&inventory(query)), None); - } - - /// Merges do not stack: in `sum(sum by (job) (sum_over_time(m[5m])))` - /// a target with a fused computation is never absorbed, and `combinations` - /// counts exactly the choices enumeration keeps. - #[test] - fn nested_sum_merges_do_not_stack() { - let inventory = inventory("sum(sum by (job) (sum_over_time(m[5m])))"); - let absorbed: Vec = absorbed_targets(&inventory.targets).collect(); - assert_eq!(absorbed.len(), 1, "one merge"); - for &a in &absorbed { - assert!( - inventory.targets[a].fused_computation.is_none(), - "an absorbed target has a fused computation" - ); - } - let enumerated = enumerated_choices(&inventory); - assert_eq!(inventory.combinations(), enumerated); - } - - /// The shapes a heap over the raw data samples cannot answer do not get one: - /// `bottomk` stays a sort and limit (no top-k target), and a grouped sum - /// `without` labels or by the metric name (which the per-series sum drops) - /// gets no heap that absorbs the per-series sum and reads the samples. - #[test] - fn grouped_range_sums_a_raw_data_samples_heap_cannot_rank_get_no_fused_heap() { - let is_topk = |inventory: &PerComputationCandidates, t: &ComputationTarget| { - matches!( - inventory.aggregate(t).non_asap(), - Some(NonASAPOp::Aggregate { measures, .. }) - if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) - ) - }; - let bottom = inventory("bottomk(3, sum by (region) (sum_over_time(m[1m])))"); - assert!( - !bottom.targets.iter().any(|t| is_topk(&bottom, t)), - "bottomk has no top-k target" - ); - for query in [ - "topk(3, sum without (instance) (sum_over_time(m[1m])))", - "topk(3, sum by (__name__) (sum_over_time(m[1m])))", - ] { - let inventory = inventory(query); - for target in inventory.targets.iter().filter(|t| is_topk(&inventory, t)) { - assert!( - target - .fused_computation - .as_ref() - .is_none_or(|f| f.absorbed_targets.len() == 1), - "{query}: {:?}", - target.fused_computation - ); - } - } - } - - /// A fused computation picked leaves every target beneath it uncomputed, - /// also one that an absorbed target's own merge absorbs: Pass 1 closes - /// `absorbed_targets` over those, so enumeration keeps only their first choice and - /// `combinations` still counts exactly the choices kept. Here the top-k - /// heap is made to absorb the grouped sum only, whose merge absorbs the - /// per-series sum. - #[test] - fn a_fused_computation_absorbs_what_its_absorbed_targets_merges_absorb() { - let mut inventory = inventory("topk(10, sum by (job) (sum_over_time(m[1m])))"); - let topk = topk_target(&inventory); - let fused = inventory.targets[topk] - .fused_computation - .as_mut() - .expect("a fused heap"); - let grouped = fused.absorbed_targets[0]; - fused.absorbed_targets.truncate(1); - let per_series = inventory.targets[grouped] - .fused_computation - .as_ref() - .expect("the grouped sum's merge") - .absorbed_targets[0]; - close_absorptions(&mut inventory.targets); - assert_eq!( - inventory.targets[topk] - .fused_computation - .as_ref() - .unwrap() - .absorbed_targets, - vec![grouped, per_series] - ); - assert_eq!(inventory.combinations(), enumerated_choices(&inventory)); - } - - /// `combinations` counts each choice once: picking a fused - /// whole-expression candidate with any but the first candidate of the - /// target it absorbs is the same candidate. An absorbed target with three - /// candidates (an approximate inner count: its accumulator, Count-Min and - /// UnivMon) gives 1 exact · 3 + 2 whole-expression = 5, not 3 · 3. - #[test] - fn combinations_skip_choices_a_whole_expression_candidate_absorbs() { - let inventory = inventory("topk by (job) (10, count_over_time(m[1m]))"); - let topk = topk_target(&inventory); - let target = &inventory.targets[topk]; - let Some(fused) = &target.fused_computation else { - panic!("a fused computation: {:?}", target.ways_to_compute) - }; - let inner = fused.absorbed_targets[0]; - let n = target.ways_to_compute.len(); - let whole = fused.ways_to_compute.len(); - let m = inventory.targets[inner].ways_to_compute.len(); - let enumerated = enumerated_choices(&inventory); - assert!(whole > 0 && m > 1, "{n} {whole} {m}"); - assert_eq!(inventory.combinations(), n * m + whole); - assert_eq!(inventory.combinations(), 5); - assert_eq!(inventory.combinations(), enumerated); - } - - /// A quantile estimate of a metric's samples carries their sign only for - /// φ in [0, 1]: outside it PromQL answers -Inf or +Inf. Pass 1 offers - /// no sketch for such a φ. - #[test] - fn a_quantile_estimate_shares_its_raw_data_samples_sign_only_inside_the_unit_interval() { - let estimate_of = |query: &str| { - let inventory = inventory(query); - (0..inventory.targets.len()) - .flat_map(|t| composed_with(&inventory, t)) - .find_map(|(_, root)| { - OperatorNode::reachable(&root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - }) - }; - let inside = estimate_of("quantile_over_time(0.5, m[1m])").expect("a sketch estimate"); - assert_eq!(estimates_raw_data_samples_of(&inside).as_deref(), Some("m")); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = inside.asap() else { - unreachable!() - }; - for q in [-1.0, 2.0] { - let outside = OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: Rc::clone(summary_input), - query: SketchStatistic::Quantile { q }, - }), - inside.schema.clone(), - ); - assert_eq!(estimates_raw_data_samples_of(&outside), None, "q = {q}"); - } - for query in [ - "quantile_over_time(-1, m[1m])", - "quantile_over_time(2, m[1m])", - ] { - assert!(estimate_of(query).is_none(), "{query}: a sketch is offered"); - } - } - - /// A quantile read from merged tumbling windows of a metric's samples is - /// still one of the samples (or between two), so it carries their sign - /// as the same quantile over the raw window does; φ outside [0, 1] - /// still carries none. - #[test] - fn a_quantile_of_merged_tumbling_windows_shares_its_raw_data_samples_sign() { - let inventory = inventory("quantile_over_time(0.99, m[100s])"); - let estimate = (0..inventory.targets.len()) - .flat_map(|t| composed_with(&inventory, t)) - .find_map(|(_, root)| { - OperatorNode::reachable(&root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - }) - .expect("a sketch estimate"); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = estimate.asap() else { - unreachable!() - }; - let Some(ASAPOp::SummaryAgg { - child: range, - family, - input, - reduction, - grouping, - filter, - }) = summary_input.asap() - else { - panic!("{summary_input:?}") - }; - let chain = Rc::new( - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(range.children()[0]), - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - tumbling_window_width_ms: 10_000, - tumbling_window_count: 10, - })) - .unwrap(), - ); - let merged = Rc::new( - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![chain], - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(), - ); - let read = |q: f64| { - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: Rc::clone(&merged), - query: SketchStatistic::Quantile { q }, - }), - estimate.schema.clone(), - ) - }; - assert_eq!( - estimates_raw_data_samples_of(&read(0.99)).as_deref(), - Some("m") - ); - assert_eq!( - sign_statistic(&fold_stats(&Reduction::by(vec![]), &Rc::new(read(0.99))).unwrap()), - Some(raw_data_samples_sign_statistic("m").as_str()) - ); - for q in [-1.0, 2.0] { - assert_eq!(estimates_raw_data_samples_of(&read(q)), None, "q = {q}"); - } - } - - /// The metric whose raw data samples, or plain sums of them, a composed node's - /// values estimate: a whole-expression heap's estimates and an exact - /// sum over the samples are; a rate or any other function is not. A - /// fold over them names its unknown sign by that metric. - #[test] - fn values_estimating_raw_data_samples_name_their_sign_by_the_metric() { - let inventory = inventory("topk by (job) (10, sum_over_time(m[1m]))"); - let topk = topk_target(&inventory); - for (candidate, root) in composed_with(&inventory, topk) { - let estimate = OperatorNode::reachable(&root) - .into_iter() - .find(|n| { - matches!( - n.asap(), - Some( - ASAPOp::SummaryEstimate { .. } - | ASAPOp::FinalizeExactAccumulator { .. } - ) - ) - }) - .unwrap(); - assert_eq!( - estimates_raw_data_samples_of(&estimate).as_deref(), - Some("m"), - "{candidate:?}" - ); - let stats = fold_stats(&Reduction::by(vec![]), &estimate).unwrap(); - assert_eq!( - asap_accuracy::test_support::sign_statistic(&stats), - Some(asap_accuracy::test_support::raw_data_samples_sign_statistic("m").as_str()) - ); - } - let rate = inventory_root("sum by (job) (rate(m[1m]))"); - let finalized = OperatorNode::reachable(&rate) - .into_iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. }))) - .collect::>(); - assert!(!finalized.is_empty()); - for node in finalized { - assert_eq!(estimates_raw_data_samples_of(&node), None); - } - let stats = fold_stats(&Reduction::by(vec![]), &rate).unwrap(); - assert_eq!(sign_statistic(&stats), None); - } - - /// The sketch parameters of every summary state below `root`. - fn sketch_params(root: &Rc) -> Vec { - OperatorNode::reachable(root) - .into_iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .collect() - } - - /// `query` composed with target `t` at the candidate `pick(t, its - /// candidates)` returns the position of. - fn composed_picking( - query: &str, - pick: impl Fn(usize, &[HowToComputeAggregateMeasure]) -> usize, - ) -> Rc { - let inventory = inventory(query); - let choice: Vec = inventory - .targets - .iter() - .enumerate() - .map(|(t, target)| pick(t, &target.ways_to_compute)) - .collect(); - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - root.clone() - } - - fn position_of( - candidates: &[HowToComputeAggregateMeasure], - candidate: HowToComputeAggregateMeasure, - ) -> usize { - candidates - .iter() - .position(|c| *c == candidate) - .expect("the candidate is offered") - } - - fn cms(eps: f64, delta: f64) -> SketchParams { - SketchParams::Cms { - width: asap_accuracy::test_support::cms_width(eps), - depth: asap_accuracy::test_support::cms_depth(delta), - } - } - - /// A combination of approximate values splits the query's budget - /// equally among them (#669b): the difference of two Count-Min counts - /// sizes each for ε/2 and δ/2 of the query's ε = 0.01, δ = 0.001. - #[test] - fn approximate_inputs_of_a_combination_share_the_budget_equally() { - let both = composed_picking("count by (job) (a) - count by (job) (b)", |_, c| { - position_of( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Cms), - ) - }); - assert!(both.guarantee.as_ref().is_some_and(|g| !g.is_exact())); - assert_eq!(sketch_params(&both), vec![cms(0.005, 0.0005); 2]); - } - - /// Only the inputs that are approximate in the combination share the - /// budget: with one count exact, the other gets all of it. - #[test] - fn an_exact_input_takes_no_share_of_the_budget() { - let one = composed_picking("count by (job) (a) - count by (job) (b)", |t, c| { - if t == 0 { - position_of( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Cms), - ) - } else { - 0 - } - }); - assert_eq!(sketch_params(&one), vec![cms(0.01, 0.001)]); - } - - /// The first composed candidate of `query`. - fn inventory_root(query: &str) -> Rc { - let inventory = inventory(query); - let composed = - compose_logical_candidate(&inventory, &vec![0; inventory.targets.len()]).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - root.clone() - } - - /// A Count-Min estimate has a bound only over weights proven - /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. - #[test] - fn count_min_estimate_has_no_bound_over_an_open_requirement() { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - asap_types::ir::schema::SketchParams::Cms { width: 8, depth: 2 }, - ), - GroupingStrategy::default(), - ); - let query = SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }; - let with = |weight_domain| SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain, - }; - assert!(!estimate_has_bound( - &family, - &with(WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }), - &query - )); - assert!(!estimate_has_bound( - &family, - &with(WeightDomain::UnknownOrSigned), - &query - )); - assert!(estimate_has_bound( - &family, - &with(WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples - }), - &query - )); - } - - /// A Count-Min estimate over samples of `m` gets its bound gated on - /// `m`'s sign: unknown until plan selection binds it, the local bound - /// once bound to 1. - #[test] - fn count_min_estimate_over_raw_data_samples_is_gated_on_the_metric() { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - asap_types::ir::schema::SketchParams::Cms { - width: 272, - depth: 5, - }, - ), - GroupingStrategy::default(), - ); - let query = SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }; - let update = SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }, - }; - let gated = estimate_bound_over_raw_data_samples(&family, &update, &query).expect("gated"); - assert_eq!(gated.spec.error.evaluate(), None); - let local = local_guarantee(&family, &query).unwrap(); - let statistic = raw_data_samples_sign_statistic("m"); - // Every other statistic (a stream norm) bound alike. - let others = |s: &str| Some(if s == statistic { 1.0 } else { 1_000.0 }); - let bound = gated.bind(&others).spec.error.evaluate(); - assert!(bound.is_some()); - assert_eq!(bound, local.bind(&others).spec.error.evaluate()); - let unconfirmed = gated - .bind(&|s| (s != statistic).then_some(1_000.0)) - .spec - .error - .evaluate(); - assert_eq!(unconfirmed, None); - } - - /// Classic HLL's failure probability is left symbolic in a source's - /// distinct bound only for one instance per group over that source's - /// own rows (#693's scope condition): not over a join of two sources, - /// not over computed rows, and not for one Hydra structure shared by - /// every group, whose distinct items the source's bound does not bound. - #[test] - fn classic_hll_confidence_needs_one_instance_over_a_sources_own_rows() { - use asap_types::ir::schema::{HydraKind, HydraParams}; - let hll = |grouping| { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 15, - relative_error: Some(0.05), - }, - ), - grouping, - ) - }; - let input = |query| crate::test_support::lower_promql(query, AccuracyTarget::exact()); - let guarantee = |family: &FieldDataType, query| { - source_distinct_guarantee(family, &SketchStatistic::Cardinality, &input(query)) - }; - let own = hll(GroupingStrategy::PerSubpopulationInstance); - let symbolic = guarantee(&own, "a").expect("a source's own rows"); - assert_eq!( - symbolic.spec.failure_probability, - ProbabilityExpr::Unknown { - statistic: asap_accuracy::estimators::hll::classic_failure_statistic(15, 0.05, "a") - } - ); - for (query, over) in [("a * on() b", "a join"), ("a * 2", "computed rows")] { - assert!(guarantee(&own, query).is_none(), "{over}: {query}"); - } - let hydra = hll(GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: 8, - depth: 2, - shared_rows: 2, - shared_columns: 8, - }, - }); - assert!(guarantee(&hydra, "a").is_none()); - } - - /// A deployment whose executor supports only `supported`: each family - /// with its readouts. - fn supporting(supported: &[(SketchAlgorithm, &[StatisticKind])]) -> Vec { - supported - .iter() - .map(|(family, readouts)| SupportedSummary { - family: *family, - readouts: readouts.to_vec(), - }) - .collect() - } - - /// A family the deployment cannot build is pruned from Pass 1 with - /// `UnsupportedByDeployment`; the supported family and the exact option - /// stay. With every summary supported the inventory is unchanged. - #[test] - fn a_family_the_deployment_cannot_build_is_pruned() { - let query = "quantile_over_time(0.99, m[5m])"; - let unrestricted = inventory(query); - let restricted = inventory(query).without_unsupported_summaries(Some(&supporting(&[( - SketchAlgorithm::Kll, - &[StatisticKind::Quantile], - )]))); - let target = &restricted.targets[0]; - assert_eq!( - target.ways_to_compute, - [ - HowToComputeAggregateMeasure::Exact, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ] - ); - assert_eq!( - restricted.pruned_of_target(0, false), - [( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch), - PruneReason::UnsupportedByDeployment - )] - ); - let default = inventory(query).without_unsupported_summaries(None); - assert_eq!( - default.targets[0].ways_to_compute, - unrestricted.targets[0].ways_to_compute - ); - } - - /// A family the deployment builds but cannot answer the measure's - /// readout from is pruned: UnivMon, supported for distinct counts and - /// L2 only, is no count candidate; Count-Min, supported for the total, - /// stays. - #[test] - fn a_readout_the_deployment_cannot_answer_prunes_its_family() { - let restricted = inventory("count_over_time(m[1m])").without_unsupported_summaries(Some( - &supporting(&[ - (SketchAlgorithm::Cms, &[StatisticKind::Total]), - ( - SketchAlgorithm::UnivMon, - &[StatisticKind::Distinct, StatisticKind::FrequencyL2], - ), - ]), - )); - let target = &restricted.targets[0]; - assert_eq!( - target.ways_to_compute, - [ - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count]), - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Cms), - ] - ); - assert_eq!( - restricted.pruned_of_target(0, false), - [( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::UnivMon), - PruneReason::UnsupportedByDeployment - )] - ); - } - - /// A fused whole-expression top-k loses the heap families the - /// deployment cannot answer top-k from, each recorded with its reason, - /// and is not offered once it has none left: the target and the inner - /// sum keep their exact options. - #[test] - fn a_fused_computation_loses_its_unsupported_heaps() { - let query = "topk by (job) (10, sum_over_time(m[1m]))"; - let one_heap = inventory(query).without_unsupported_summaries(Some(&supporting(&[( - SketchAlgorithm::CountSketchWithHeap, - &[StatisticKind::TopK], - )]))); - let topk = topk_target(&one_heap); - assert_eq!( - one_heap.targets[topk] - .fused_computation - .as_ref() - .unwrap() - .ways_to_compute, - [HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::CountSketchWithHeap - )] - ); - assert_eq!( - one_heap.pruned_of_target(topk, true), - [( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - PruneReason::UnsupportedByDeployment - )] - ); - assert_eq!(one_heap.combinations(), 2); - // Count-Min with a heap answers item frequencies, not top-k. - let none = inventory(query).without_unsupported_summaries(Some(&supporting(&[( - SketchAlgorithm::CmsWithHeap, - &[StatisticKind::ItemFrequency], - )]))); - let topk = topk_target(&none); - let fused = none.targets[topk].fused_computation.as_ref().unwrap(); - assert!(fused.ways_to_compute.is_empty()); - assert_eq!( - none.pruned_of_target(topk, true), - [ - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - PruneReason::UnsupportedByDeployment - ), - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - PruneReason::UnsupportedByDeployment - ), - ] - ); - assert_eq!( - none.targets[topk].ways_to_compute, - [HowToComputeAggregateMeasure::Exact] - ); - assert_eq!(none.combinations(), 1); - } - - /// Mergeable intents give exactly their accumulator (`avg`: sum, count); other exact-only intents give `Exact`. - #[test] - fn intent_candidate_matrix() { - use asap_types::ir::operator::agg_intent::{MathFunc, TimeFunc}; - use AggIntent as A; - use ExactKind as E; - let accumulator = |kinds: Vec| HowToComputeAggregateMeasure::ExactAccumulator(kinds); - let exact_count = A::Count { - accuracy: AccuracyTarget::exact(), - }; - let mut matrix: Vec<(A, ComputationChoices)> = [ - (A::Min { col: None }, vec![E::Min]), - (A::Max { col: None }, vec![E::Max]), - (A::IRate, vec![E::IRate]), - (A::Increase, vec![E::Increase]), - (A::Sum { col: None }, vec![E::Sum]), - (A::Rate, vec![E::Rate]), - (A::Avg { col: None }, vec![E::Sum, E::Count]), - ] - .into_iter() - .map(|(intent, kinds)| { - ( - intent, - ComputationChoices { - candidates: vec![accumulator(kinds)], - pruned: vec![], - }, - ) - }) - .collect(); - // An exact count prunes its summaries for requiring an exact answer. - matrix.push(( - exact_count.clone(), - ComputationChoices { - candidates: vec![accumulator(vec![E::Count])], - pruned: summary_candidates(&exact_count) - .iter() - .map(|a| { - ( - HowToComputeAggregateMeasure::Summary(*a), - PruneReason::RequiresExact, - ) - }) - .collect(), - }, - )); - let exact_only = [ - A::PearsonCorr { left: 0, right: 1 }, - A::Extension { - ext_kind: "frequency".into(), - payload: serde_json::json!({ "item": "checkout" }), - }, - A::HistogramQuantile { q: 0.99, le: 0 }, - A::Changes, - A::Delta, - A::IDelta, - A::Deriv, - A::Resets, - A::PredictLinear { seconds: 60.0 }, - A::DoubleExpSmoothing { - smoothing: 0.5, - trend: 0.5, - }, - A::HistogramCount, - A::HistogramSum, - A::HistogramAvg, - A::HistogramStdDev, - A::HistogramStdVar, - A::HistogramFraction { - lower: 0.0, - upper: 1.0, - }, - A::Math(MathFunc::Abs), - A::TimeFn(TimeFunc::Hour), - A::Absent { - labels: Default::default(), - }, - A::AbsentOverTime, - A::PresentOverTime, - A::Group, - A::CountValues { label: "v".into() }, - A::LastOverTime, - A::FirstOverTime, - A::MadOverTime, - A::TsOfMinOverTime, - A::TsOfMaxOverTime, - A::TsOfFirstOverTime, - A::TsOfLastOverTime, - ]; - matrix.extend(exact_only.into_iter().map(|intent| { - ( - intent, - ComputationChoices { - candidates: vec![HowToComputeAggregateMeasure::Exact], - pruned: vec![], - }, - ) - })); - for (intent, expected) in &matrix { - assert_eq!( - &computation_candidates_for_intent(intent).unwrap(), - expected, - "{intent:?}" - ); - } - // Correlation never gets a single-input summary. - assert!(summary_candidates(&A::PearsonCorr { left: 0, right: 1 }).is_empty()); - } - - /// `avg_over_time` divides with a finite-division guard; a grouped `avg` does not. - #[test] - fn temporal_average_divides_with_a_finite_division_guard() { - let division_guard = |query: &str| { - let root = inventory_root(query); - let divisions: Vec = OperatorNode::reachable(&root) - .iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::BinaryOp { operator, .. }) - if operator.kind == BinaryOpKind::Arithmetic(ArithmeticOpKind::Div) => - { - Some(operator.checked_finite_division) - } - _ => None, - }) - .collect(); - let [guarded] = divisions[..] else { - panic!("{query}: one sum / count division: {root:#?}") - }; - guarded - }; - assert!(division_guard("avg_over_time(a[5m])")); - assert!(!division_guard("avg by (job) (m)")); - } - - /// The candidates of `query` lowered at `accuracy`, over rows without the - /// series-identity column (open PromQL schemas): over identity-carrying - /// (closed) rows a grouped top-k gets no whole-expression heap. - fn inventory_at(query: &str, accuracy: AccuracyTarget) -> PerComputationCandidates { - let root = crate::test_support::lower_promql(query, accuracy); - generate_per_computation_candidates(&crate::stage0::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ])) - .unwrap() - } - - fn open_inventory(query: &str) -> PerComputationCandidates { - inventory_at( - query, - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ) - } - - /// The estimate and state of the top-k's fused whole-expression heap of - /// `algorithm`. - fn fused_heap( - inventory: &PerComputationCandidates, - algorithm: SketchAlgorithm, - ) -> (SketchStatistic, Rc) { - let topk = topk_target(inventory); - let own = inventory.targets[topk].ways_to_compute.len(); - let (_, root) = composed_with(inventory, topk) - .into_iter() - .skip(own) - .find(|(candidate, _)| *candidate == HowToComputeAggregateMeasure::Summary(algorithm)) - .unwrap_or_else(|| panic!("a fused {algorithm:?} candidate")); - let [state] = &heap_states(&root)[..] else { - panic!("one heap: {root:#?}") - }; - let estimate = OperatorNode::reachable(&root) - .into_iter() - .find_map(|node| match node.asap() { - Some(ASAPOp::SummaryEstimate { - summary_input, - query, - }) if Rc::ptr_eq(summary_input, state) => Some(query.clone()), - _ => None, - }) - .expect("the heap's estimate"); - (estimate, state.clone()) - } - - fn heap_update(state: &OperatorNode) -> &SummaryUpdate { - match state.asap() { - Some(ASAPOp::SummaryAgg { input, .. }) => input, - _ => unreachable!(), - } - } - - /// A top-k over a `count` fuses into one global unit-weight heap over the raw rows, ranking the label. - #[test] - fn count_ranked_topk_fuses_to_one_global_heap_sketch() { - let inventory = open_inventory("topk(10, count by (service) (m))"); - let topk = topk_target(&inventory); - let fused = inventory.targets[topk] - .fused_computation - .as_ref() - .expect("a fused heap"); - assert!(fused - .ways_to_compute - .contains(&HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::CmsWithHeap - ))); - let (estimate, state) = fused_heap(&inventory, SketchAlgorithm::CmsWithHeap); - assert_eq!(estimate, SketchStatistic::TopK { k: 10 }); - let update = heap_update(&state); - assert_eq!( - update.item, - Some(SummaryInputExpr::Column(ColumnRef::Named("service".into()))) - ); - assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); - assert_eq!( - update.weight_domain, - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - } - ); - let Some(ASAPOp::SummaryAgg { child, .. }) = state.asap() else { - unreachable!() - }; - assert!(!child.contains_asap()); - } - - /// The ranked item over several group keys is their tuple, in the lowered (name-sorted) key order. - #[test] - fn multidimensional_ranked_item_is_a_structured_tuple() { - let inventory = open_inventory("topk(10, count by (service, region) (m))"); - let (_, state) = fused_heap(&inventory, SketchAlgorithm::CmsWithHeap); - let column = |name: &str| SummaryInputExpr::Column(ColumnRef::Named(name.into())); - assert_eq!( - heap_update(&state).item, - Some(SummaryInputExpr::Tuple(vec![ - column("region"), - column("service") - ])) - ); - } - - /// The top-k's own partition keys partition the heap and are left out of the ranked tuple. - #[test] - fn subpopulation_columns_are_not_part_of_the_ranked_item_tuple() { - let inventory = - open_inventory("topk by (region) (10, count by (service, method, region) (m))"); - let (_, state) = fused_heap(&inventory, SketchAlgorithm::CmsWithHeap); - let column = |name: &str| SummaryInputExpr::Column(ColumnRef::Named(name.into())); - assert_eq!( - heap_update(&state).item, - Some(SummaryInputExpr::Tuple(vec![ - column("method"), - column("service") - ])) - ); - let Some(ASAPOp::SummaryAgg { - child, reduction, .. - }) = state.asap() - else { - unreachable!() - }; - let keys: Vec<_> = reduction - .group_keys() - .expect("partitioned by region") - .keys() - .iter() - .map(|&k| child.schema.fields[k].name.as_str()) - .collect(); - assert_eq!(keys, ["region"]); - } - - /// A per-series top-k ranks series identity (column, else label set) weighted by sample value. - #[test] - fn temporal_per_entity_topk_uses_series_identity_and_sample_value() { - let query = "topk by (service) (5, sum_over_time(m[1m]))"; - let with_identity = inventory(query); - let without_identity = open_inventory(query); - for (inventory, item) in [ - ( - with_identity, - SummaryInputExpr::Column(ColumnRef::Named( - asap_types::ir::schema::PROMQL_SERIES_IDENTITY.into(), - )), - ), - ( - without_identity, - SummaryInputExpr::EntityIdentity(EntityIdentity::PromqlLabelSet { - excluding: vec![ColumnRef::Named("service".into())], - }), - ), - ] { - let (_, state) = fused_heap(&inventory, SketchAlgorithm::CountSketchWithHeap); - let update = heap_update(&state); - assert_eq!(update.item, Some(item)); - assert_eq!( - update.weight, - SummaryInputExpr::Column(ColumnRef::SampleValue) - ); - } - } - - // Identity realization must preserve named labels and every top-k candidate's output contract. - #[test] - fn topk_keeps_known_labels_beside_series_identity() { - let inventory = inventory("topk by (service) (5, sum_over_time(m{instance=\"a\"}[1m]))"); - let topk = topk_target(&inventory); - let expected = &inventory.aggregate(&inventory.targets[topk]).schema; - assert!(expected.fields.iter().any(|field| field.name == "instance")); - assert!(expected - .fields - .iter() - .any(|field| field.name == asap_types::ir::schema::PROMQL_SERIES_IDENTITY)); - for (_, root) in composed_with(&inventory, topk) { - assert_eq!(root.schema.fields, expected.fields); - assert_eq!(root.schema.time_index, expected.time_index); - assert_eq!(root.schema.closed, expected.closed); - // A physical heap may prove an additional partition/item key. - // Every required uniqueness fact must still follow from its keys. - for required in &expected.unique_keys { - assert!( - root.schema.unique_keys.iter().any(|supplied| { - supplied.iter().all(|column| required.contains(column)) - }), - "required key {required:?} is not implied by {:?}", - root.schema.unique_keys - ); - } - } - let (_, state) = fused_heap(&inventory, SketchAlgorithm::CountSketchWithHeap); - assert!(matches!( - heap_update(&state).item, - Some(SummaryInputExpr::Tuple(_)) - )); - } - - /// A KLL quantile over an exact sum accumulator composes per node, the sum adding no error. - #[test] - fn nested_quantile_over_an_exact_sum_composes_per_node() { - let sum = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); - let kll = HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll); - let inventory = inventory_at( - "quantile(0.9, sum by (job) (m))", - AccuracyTarget::max_error(0.01).failure_probability(0.01), - ); - let choice: Vec = inventory - .targets - .iter() - .map(|target| { - let candidates = &target.ways_to_compute; - if candidates.contains(&kll) { - position_of(candidates, kll.clone()) - } else { - position_of(candidates, sum.clone()) - } - }) - .collect(); - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - let estimate = OperatorNode::reachable(root) - .into_iter() - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - .expect("a quantile estimate"); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = estimate.asap() else { - unreachable!() - }; - let Some(ASAPOp::SummaryAgg { child, family, .. }) = summary_input.asap() else { - panic!("a KLL state: {summary_input:#?}") - }; - let FieldDataType::Sketch(kind, _) = family else { - panic!("a sketch: {family:?}") - }; - assert_eq!(kind.algorithm(), &SketchAlgorithm::Kll); - assert_eq!( - kind.params(), - &SketchParams::Kll { - k: 269, - failure_probability: None - } - ); - let Some(ASAPOp::FinalizeExactAccumulator { child: sum_state }) = child.asap() else { - panic!("the finalized sum: {child:#?}") - }; - let Some(ASAPOp::SummaryAgg { - family: sum_family, - child: leaf, - .. - }) = sum_state.asap() - else { - panic!("the sum's accumulator: {sum_state:#?}") - }; - assert_eq!( - sum_family, - &FieldDataType::ExactAggregate( - ExactKind::Sum, - asap_types::ir::schema::ExactParams::Sum - ) - ); - assert!(!leaf.contains_asap()); - assert!(child - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact)); - let guarantee = root.guarantee.as_ref().expect("a rank bound"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Rank); - assert_eq!( - guarantee.spec.error.evaluate(), - Some(asap_accuracy::test_support::kll_rank_error_99(269)) - ); - } - - /// Pass 1's inventory of `queries`, each lowered with its target, less - /// the summaries no query can use over `data`. - fn usable_inventory( - queries: &[(&str, AccuracyTarget)], - data: &DataWorkload, - ) -> PerComputationCandidates { - let roots: Vec = queries - .iter() - .map(|(query, target)| { - let root = crate::test_support::lower_promql(query, target.clone()); - QueryRoot::Operator(root) - }) - .collect(); - let targets: Vec = - queries.iter().map(|(_, target)| target.clone()).collect(); - generate_per_computation_candidates(&crate::stage0::generate_dag_of_logical_memo_groups( - &roots, - )) - .unwrap() - .without_summaries_no_query_can_use( - &targets, - data, - &asap_accuracy::BuiltinAccuracyModel, - ) - } - - /// A per-value target at ε 0.05, δ 0.01 in `metric`. - fn target_in(metric: AccuracyMetric) -> AccuracyTarget { - AccuracyTarget::max_error(0.05) - .failure_probability(0.01) - .accuracy_metric(metric) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - } - - /// Metric `x`, its samples declared non-negative or not. - fn x_data(non_negative: bool) -> DataWorkload { - let mut data = - DataWorkload::sampled_every(["x".to_string()], asap_types::workload::DurationMs(1_000)); - data.sources.get_mut("x").unwrap().values_non_negative = non_negative; - data - } - - /// The summary families pruned for `reason`, and those still offered. - fn pruned_and_offered( - inventory: &PerComputationCandidates, - reason: PruneReason, - ) -> (Vec, Vec) { - let summary = |way: &HowToComputeAggregateMeasure| match way { - HowToComputeAggregateMeasure::Summary(algorithm) => Some(*algorithm), - _ => None, - }; - let pruned = inventory - .pruned_ways_to_compute - .iter() - .filter(|pruned| pruned.reason == reason) - .filter_map(|pruned| summary(&pruned.way_to_compute)) - .collect(); - let offered = inventory - .targets - .iter() - .flat_map(|target| &target.ways_to_compute) - .filter_map(summary) - .collect(); - (pruned, offered) - } - - /// An average of per-series p99s at a relative target: a relative error - /// passes through `avg` only over samples declared non-negative, and a - /// rank error never meets a relative target. Over an undeclared gauge - /// every quantile sketch is pruned with - /// [`PruneReason::NoErrorBoundForQueryTarget`]; declared non-negative, - /// DDSketch is kept and KLL still pruned. At a rank target, a rank - /// error has no rule through `avg`, so KLL is pruned too. - #[test] - fn a_relative_bound_through_avg_is_pruned_without_a_sign_declaration() { - let query = "avg by (g) (quantile_over_time(0.99, x[100s]))"; - let reason = PruneReason::NoErrorBoundForQueryTarget; - let relative = target_in(AccuracyMetric::Relative); - let undeclared = usable_inventory(&[(query, relative.clone())], &x_data(false)); - let (pruned, offered) = pruned_and_offered(&undeclared, reason); - assert!(pruned.contains(&SketchAlgorithm::DDSketch), "{pruned:?}"); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - assert!(offered.is_empty(), "{offered:?}"); - let declared = usable_inventory(&[(query, relative)], &x_data(true)); - let (pruned, offered) = pruned_and_offered(&declared, reason); - assert_eq!(offered, vec![SketchAlgorithm::DDSketch], "{pruned:?}"); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - let rank = usable_inventory(&[(query, target_in(AccuracyMetric::Rank))], &x_data(true)); - let (pruned, offered) = pruned_and_offered(&rank, reason); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - assert!(offered.is_empty(), "{offered:?}"); - } - - /// A summary one of the queries reading its group can use is kept: the - /// per-series p99s are both a query's result, which a KLL rank bound - /// meets, and the input of an average, which no rank bound reaches. - #[test] - fn a_summary_one_of_two_queries_can_use_is_kept() { - let rank = target_in(AccuracyMetric::Rank); - let alone = usable_inventory( - &[("avg(quantile_over_time(0.99, x[5m]))", rank.clone())], - &x_data(false), - ); - let (pruned, _) = pruned_and_offered(&alone, PruneReason::NoErrorBoundForQueryTarget); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - let shared = usable_inventory( - &[ - ("quantile_over_time(0.99, x[5m])", rank.clone()), - ("avg(quantile_over_time(0.99, x[5m]))", rank), - ], - &x_data(false), - ); - let (pruned, offered) = - pruned_and_offered(&shared, PruneReason::NoErrorBoundForQueryTarget); - assert!(offered.contains(&SketchAlgorithm::Kll), "{offered:?}"); - assert!(!pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - } - - /// Exact candidates are never pruned: every target keeps its exact - /// option first, and only summaries are pruned for the target. - #[test] - fn exact_candidates_are_never_pruned_for_the_target() { - let rank = target_in(AccuracyMetric::Rank); - for query in [ - "avg by (g) (quantile_over_time(0.99, x[100s]))", - "avg(quantile_over_time(0.99, x[5m])) > 3", - "sum(count_over_time(x[5m])) / avg(quantile_over_time(0.5, x[5m]))", - ] { - let inventory = usable_inventory(&[(query, rank.clone())], &x_data(false)); - assert!(!inventory.targets.is_empty(), "{query}"); - for target in &inventory.targets { - assert!( - matches!( - target.ways_to_compute.first(), - Some( - HowToComputeAggregateMeasure::Exact - | HowToComputeAggregateMeasure::ExactAccumulator(_) - ) - ), - "{query}: {:?}", - target.ways_to_compute - ); - } - assert!( - inventory - .pruned_ways_to_compute - .iter() - .filter(|pruned| pruned.reason == PruneReason::NoErrorBoundForQueryTarget) - .all(|pruned| matches!( - pruned.way_to_compute, - HowToComputeAggregateMeasure::Summary(_) - )), - "{query}" - ); - } - } - - /// A summary another aggregate over the same source also offers is - /// kept, though no query reading its own group can use it: Pass 2 may - /// build the other group's state from it (a rollup or window rollup), - /// which Pass 1 does not see. - #[test] - fn a_summary_pass2_may_reuse_for_another_aggregate_is_kept() { - let rank = target_in(AccuracyMetric::Rank); - let reason = PruneReason::NoErrorBoundForQueryTarget; - let with_other = usable_inventory( - &[ - ("avg(quantile_over_time(0.99, x[5m]))", rank.clone()), - ("quantile_over_time(0.99, x[10m])", rank), - ], - &x_data(false), - ); - let (pruned, _) = pruned_and_offered(&with_other, reason); - assert!(!pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - } - - /// A whole-result target with a failure probability over a number of - /// values the data does not bound cannot be met by a per-value - /// guarantee that may fail, whatever its size: KLL is pruned. With the - /// series count declared, it is kept. DDSketch's relative bound never - /// fails, so it is not pruned for the scope (only for the metric under - /// a rank target). - #[test] - fn a_whole_result_target_over_unbound_values_prunes_a_failing_guarantee() { - let whole = target_in(AccuracyMetric::Rank) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::WholeResult); - let query = "quantile_over_time(0.99, x[5m])"; - let reason = PruneReason::NoErrorBoundForQueryTarget; - let unbound = usable_inventory(&[(query, whole.clone())], &x_data(false)); - let (pruned, _) = pruned_and_offered(&unbound, reason); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - let bound = usable_inventory(&[(query, whole)], &x_data(false).with_series_count(10_000)); - let (pruned, offered) = pruned_and_offered(&bound, reason); - assert!(offered.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - } - - /// The reuse escape keeps a summary only for another target whose own - /// queries can use the family: two averages of p99s over different - /// windows of one metric at a rank target both offer KLL, and neither - /// can use it (a rank error has no rule through `avg`), so it is - /// pruned for both. Before, each kept it for the other. - #[test] - fn a_summary_no_reusing_target_can_use_is_pruned() { - let rank = target_in(AccuracyMetric::Rank); - let inventory = usable_inventory( - &[ - ("avg(quantile_over_time(0.99, x[5m]))", rank.clone()), - ("avg(quantile_over_time(0.99, x[10m]))", rank), - ], - &x_data(false), - ); - let (pruned, offered) = - pruned_and_offered(&inventory, PruneReason::NoErrorBoundForQueryTarget); - assert_eq!( - pruned - .iter() - .filter(|&&family| family == SketchAlgorithm::Kll) - .count(), - 2, - "{pruned:?}" - ); - assert!(!offered.contains(&SketchAlgorithm::Kll), "{offered:?}"); - } - - /// Classic HLL is pruned when its source's declared distinct bound is - /// outside the contract at every precision (above 2^17 distinct items): - /// its failure probability is 1 at any size. Under the bound it is kept, - /// and with no declared bound too (plan selection may bind one). Two - /// queries over the same metric do not keep it for each other. - #[test] - fn hll_over_a_distinct_bound_outside_its_contract_is_pruned() { - let relative = target_in(AccuracyMetric::Relative); - let queries = [ - ("distinct_over_time(x[5m])", relative.clone()), - ("distinct_over_time(x[10m])", relative), - ]; - let with_bound = |bound: Option| { - let mut data = x_data(true); - data.sources - .get_mut("x") - .unwrap() - .max_distinct_per_evaluation = bound; - data - }; - let reason = PruneReason::NoErrorBoundForQueryTarget; - let (_, offered) = - pruned_and_offered(&usable_inventory(&queries, &with_bound(None)), reason); - assert!(offered.contains(&SketchAlgorithm::Hll), "{offered:?}"); - let (_, offered) = pruned_and_offered( - &usable_inventory(&queries, &with_bound(Some(1_000))), - reason, - ); - assert!(offered.contains(&SketchAlgorithm::Hll), "{offered:?}"); - let (pruned, offered) = pruned_and_offered( - &usable_inventory(&queries, &with_bound(Some(1_000_000))), - reason, - ); - assert!(pruned.contains(&SketchAlgorithm::Hll), "{pruned:?}"); - assert!(!offered.contains(&SketchAlgorithm::Hll), "{offered:?}"); - } -} diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs deleted file mode 100644 index 87189af51..000000000 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs +++ /dev/null @@ -1,1390 +0,0 @@ -//! Frontend-to-Pass-1 acceptance: candidate discovery precedes empirical selection. -use crate::pass1::per_computation_candidates::{ - computation_candidates_for_intent, generate_per_computation_candidates, - PerComputationCandidateError, -}; -use crate::stage0::generate_dag_of_logical_memo_groups; -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::{ - DataType, ExactKind, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, - SketchKind, SketchParams, SummaryUpdate, -}; -use asap_types::ir::{ - ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot, ScalarExpr, -}; -use asap_types::stages::PruneReason; -use asap_types::stages::{HavingEntry, HavingPlacement, HowToComputeAggregateMeasure}; -use std::rc::Rc; - -fn approximate() -> AccuracyTarget { - AccuracyTarget::max_error(0.05).failure_probability(0.01) -} -fn candidates_for(intent: AggIntent) -> Vec { - computation_candidates_for_intent(&intent) - .unwrap() - .candidates -} -fn algorithms(choices: &[HowToComputeAggregateMeasure]) -> Vec { - choices - .iter() - .filter_map(|choice| match choice { - HowToComputeAggregateMeasure::Summary(algorithm) => Some(*algorithm), - _ => None, - }) - .collect() -} -fn scan(nullable_value: bool) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "flows".into(), - }, - predicates: vec![], - schema: Schema::lifted( - vec![ - Field::plain("src_ip", DataType::Utf8, false), - Field::plain("value", DataType::Float64, nullable_value), - ], - None, - ), - })) - .unwrap() -} -fn aggregate_over( - child: Rc, - measures: Vec, - filters: Vec>, - having: Option, -) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child, - reduction: Reduction::by(vec![]), - measures, - output_names: vec![], - filters, - having, - })) - .unwrap() -} -fn aggregate(intent: AggIntent) -> Rc { - aggregate_over(scan(false), vec![intent], vec![], None) -} -fn always() -> Predicate { - Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))) -} - -/// Example 2 preserves specialized distinct summaries and the universal alternative. -#[test] -fn cardinality_keeps_exact_specialized_and_universal_candidates() { - let choices = candidates_for(AggIntent::Cardinality { - cols: vec![0], - accuracy: approximate(), - }); - assert_eq!(choices[0], HowToComputeAggregateMeasure::Exact); - assert_eq!( - algorithms(&choices), - vec![ - SketchAlgorithm::Hll, - SketchAlgorithm::Kmv, - SketchAlgorithm::UnivMon - ] - ); - let tuple = candidates_for(AggIntent::Cardinality { - cols: vec![0, 1], - accuracy: approximate(), - }); - assert!(!algorithms(&tuple).contains(&SketchAlgorithm::UnivMon)); -} - -/// Frequency moments retain exact execution and a universal sketch without certification. -#[test] -fn frequency_statistics_keep_universal_choices() { - for intent in [ - AggIntent::FrequencyL2 { - col: Some(0), - accuracy: approximate(), - }, - AggIntent::FrequencyEntropy { - col: Some(0), - accuracy: approximate(), - }, - ] { - let choices = candidates_for(intent); - assert_eq!(choices[0], HowToComputeAggregateMeasure::Exact); - assert_eq!(algorithms(&choices), vec![SketchAlgorithm::UnivMon]); - } -} - -/// An exact request cannot acquire an approximate sketch merely because one is -/// available; each pruned family records why. -#[test] -fn exact_quantile_stays_exact_and_approximate_keeps_both_families() { - let choices = candidates_for(AggIntent::Quantile { - col: Some(0), - q: 0.99, - accuracy: approximate(), - }); - assert_eq!( - algorithms(&choices), - vec![SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] - ); - let exact = computation_candidates_for_intent(&AggIntent::Quantile { - col: Some(0), - q: 0.99, - accuracy: AccuracyTarget::exact(), - }) - .unwrap(); - assert_eq!(exact.candidates, vec![HowToComputeAggregateMeasure::Exact]); - assert_eq!( - exact.pruned, - vec![ - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll), - PruneReason::RequiresExact - ), - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch), - PruneReason::RequiresExact - ), - ] - ); -} - -/// Scalar roots expose their producer targets; repeated references retain one target identity. -#[test] -fn scalar_root_producers_are_discovered_once() { - let producer = aggregate(AggIntent::Cardinality { - cols: vec![0], - accuracy: approximate(), - }); - let roots = vec![ - QueryRoot::Scalar(ScalarExpr::ScalarSubquery(producer.clone())), - QueryRoot::Operator(producer.clone()), - ]; - let candidates = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); - assert_eq!(candidates.roots.len(), 2); - for root in &candidates.roots { - root.validate_structure().unwrap(); - } - assert_eq!(candidates.targets.len(), 1); - assert_eq!(**candidates.aggregate(&candidates.targets[0]), *producer); - assert!(producer.timing.is_none()); - assert!(producer.guarantee.is_none()); - producer.validate_structure().unwrap(); -} - -/// #509 Example 1 Q1 is exact: each target's one exact option is its exact -/// accumulator state (grouped sum, rate); no plain exact form and no summary. -/// One PromQL query with the default exact target, lowered through the -/// workload frontend. -fn exact_promql(query: &str) -> Vec { - use asap_types::workload::{ - BatchEntry, PlanningWorkload, Query, QueryLanguage, QueryRequirements, QueryWorkload, - }; - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy: AccuracyTarget::exact(), - ..Default::default() - }, - predictability: Default::default(), - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(asap_types::workload::DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - asap_types::workload::DurationMs(1000), - )), - }; - asap_frontend_promql::lower_promql_query_workload(&workload).unwrap() -} - -#[test] -fn example_1_exact_q1_keeps_exact_accumulators_only() { - let roots = exact_promql("sum by (job) (rate(http_requests_total[1m]))"); - let candidates = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); - let found: Vec<_> = candidates - .targets - .iter() - .map(|target| target.ways_to_compute.clone()) - .collect(); - assert_eq!( - found, - vec![ - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum - ])], - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Rate - ])], - ] - ); - assert!(candidates - .targets - .iter() - .all(|target| candidates.aggregate(target).timing.is_none())); -} - -/// Physical annotations, ASAP operators and invalid probability requirements -/// fail at the stage boundary. -#[test] -fn assigned_physical_annotation_asap_input_and_invalid_accuracy_are_rejected() { - let mut producer = (*aggregate(AggIntent::Count { - accuracy: approximate(), - })) - .clone(); - producer.timing = Some(asap_types::ir::properties::ExecutionTiming::QueryTime); - assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(Rc::new(producer.clone())) - ])), - Err(PerComputationCandidateError::AssignedPhysicalAnnotation) - )); - let mut stored = (*aggregate(AggIntent::Count { - accuracy: approximate(), - })) - .clone(); - stored.materialization = - asap_types::physical::Materialization::MatAtQuery(asap_types::physical::Medium::Memory); - assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(Rc::new(stored.clone())) - ])), - Err(PerComputationCandidateError::AssignedPhysicalAnnotation) - )); - // An annotated node below an unannotated root is rejected too. - for annotated in [producer, stored] { - let root = having_filter(Rc::new(annotated), always()); - assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root) - ])), - Err(PerComputationCandidateError::AssignedPhysicalAnnotation) - )); - } - let summary = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: scan(false), - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap(); - assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(summary) - ])), - Err(PerComputationCandidateError::ContainsAsap) - )); - let epsilon_delta = - |epsilon, delta| AccuracyTarget::max_error(epsilon).failure_probability(delta); - for target in [ - AccuracyTarget::max_error(f64::NAN), - epsilon_delta(0.0, 0.01), - epsilon_delta(-0.1, 0.01), - epsilon_delta(1.0, 0.01), - epsilon_delta(f64::INFINITY, 0.01), - epsilon_delta(0.1, 0.0), - epsilon_delta(0.1, 1.0), - ] { - assert!( - matches!( - computation_candidates_for_intent(&AggIntent::Count { - accuracy: target.clone() - }), - Err(PerComputationCandidateError::InvalidAccuracy { - root: None, - measure: None - }) - ), - "{target:?}" - ); - } - let invalid = aggregate_over( - scan(false), - vec![ - AggIntent::Count { - accuracy: approximate(), - }, - AggIntent::Count { - accuracy: epsilon_delta(1.5, 0.01), - }, - ], - vec![], - None, - ); - let error = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(invalid), - ])) - .unwrap_err(); - assert!( - matches!( - &error, - PerComputationCandidateError::InvalidAccuracy { - root: Some(root), - measure: Some(1) - } if root == "0" - ), - "{error:?}" - ); - assert!(error.to_string().contains("root 0, measure 1 "), "{error}"); -} - -/// TopK keeps both declared heap substrates without choosing an implementation. -#[test] -fn topk_keeps_both_specialized_heap_choices() { - let choices = candidates_for(AggIntent::TopK { - k: 10, - accuracy: approximate(), - }); - assert_eq!(choices[0], HowToComputeAggregateMeasure::Exact); - assert_eq!( - algorithms(&choices), - vec![ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap - ] - ); -} - -/// Summary candidates are unsized, so they do not depend on the requested error. -#[test] -fn summary_candidates_are_unsized() { - let loose = candidates_for(AggIntent::Count { - accuracy: approximate(), - }); - let tight = candidates_for(AggIntent::Count { - accuracy: AccuracyTarget::max_error(0.0001).failure_probability(0.0001), - }); - assert_eq!(loose, tight); -} - -/// Each measure of a multi-measure aggregate is its own target; a measure whose -/// only realization is its exact accumulator is still a target. -#[test] -fn multi_measure_aggregate_has_one_target_per_measure() { - let node = aggregate_over( - scan(false), - vec![ - AggIntent::Sum { col: Some(1) }, - AggIntent::Count { - accuracy: approximate(), - }, - ], - vec![], - None, - ); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(node), - ])) - .unwrap(); - let candidates = &inventory.targets; - assert_eq!( - candidates - .iter() - .map(|target| target.measure_index) - .collect::>(), - vec![0, 1] - ); - assert_eq!( - candidates[0].ways_to_compute, - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum - ])] - ); - assert_sql_count_choices(&choices_of(&inventory, 1)); -} - -/// An aggregate with no realization besides exact execution is not a target. -#[test] -fn aggregate_without_realization_is_not_a_target() { - let node = aggregate(AggIntent::StdDev { - col: Some(1), - population: true, - }); - assert!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(node) - ])) - .unwrap() - .targets - .is_empty() - ); -} - -/// A filtered SQL count keeps its executable accumulator and records its -/// filter, which composition applies to the state. -#[test] -fn filtered_measure_keeps_candidates_and_records_its_filter() { - let count = || AggIntent::Count { - accuracy: approximate(), - }; - let filtered = aggregate_over( - scan(false), - vec![count(), count()], - vec![Some(always()), None], - None, - ); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(filtered), - ])) - .unwrap(); - let targets = &inventory.targets; - assert_eq!(targets.len(), 2); - for (index, target) in targets.iter().enumerate() { - assert_sql_count_choices(&choices_of(&inventory, index)); - assert!(target.having_filters.is_empty()); - } - assert_eq!(targets[0].measure_filter, Some(always())); - assert_eq!(targets[1].measure_filter, None); -} - -/// Target `target`'s own ways to compute and the ones Pass 1 pruned. -fn choices_of( - inventory: &crate::pass1::per_computation_candidates::PerComputationCandidates, - target: usize, -) -> crate::pass1::per_computation_candidates::ComputationChoices { - crate::pass1::per_computation_candidates::ComputationChoices { - candidates: inventory.targets[target].ways_to_compute.clone(), - pruned: inventory.pruned_of_target(target, false), - } -} - -fn assert_sql_count_choices( - choices: &crate::pass1::per_computation_candidates::ComputationChoices, -) { - assert_eq!( - choices.candidates, - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Count - ])] - ); - assert_eq!( - choices.pruned, - [SketchAlgorithm::Cms, SketchAlgorithm::UnivMon] - .into_iter() - .map(|algorithm| ( - HowToComputeAggregateMeasure::Summary(algorithm), - PruneReason::NoRelationalCountSketchRealization, - )) - .collect::>() - ); -} - -fn quantile() -> AggIntent { - AggIntent::Quantile { - col: Some(1), - q: 0.5, - accuracy: approximate(), - } -} - -fn grouped_quantile(having: Option) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction: Reduction::by(vec![0]), - measures: vec![quantile()], - output_names: vec![], - filters: vec![], - having, - })) - .unwrap() -} - -/// `count(*) by (src_ip)`, the form the SQL frontend gives `GROUP BY src_ip`. -fn grouped_count(having: Option) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction: Reduction::by(vec![0]), - measures: vec![AggIntent::Count { - accuracy: approximate(), - }], - output_names: vec![], - filters: vec![], - having, - })) - .unwrap() -} -/// `HAVING` as the SQL frontend lowers it: a `Filter` above the aggregate. -fn having_filter(child: Rc, pred: Predicate) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { pred, child })).unwrap() -} -/// `column = 'a'` over the aggregate output (0: `src_ip`, 1: the count). -fn equals(column: usize, value: ScalarValue) -> Predicate { - Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(column)), - op: CompareOpKind::Eq, - right: Box::new(ScalarExpr::Literal(value)), - semantics: ExprSemantics::Sql, - }) -} - -/// A `HAVING` over group keys only is pushed below the aggregate and keeps -/// every executable candidate. -#[test] -fn key_only_having_is_pushed_below_and_keeps_candidates() { - let node = grouped_count(None); - let root = having_filter(node.clone(), equals(0, ScalarValue::Utf8("a".into()))); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root.clone()), - ])) - .unwrap(); - let targets = &inventory.targets; - assert_eq!(targets.len(), 1); - assert_eq!(**inventory.aggregate(&targets[0]), *node); - assert_eq!( - targets[0].having_filters, - vec![HavingEntry { - filter: Some(root.clone()), - predicate: equals(0, ScalarValue::Utf8("a".into())), - placement: HavingPlacement::BelowAggregate, - }] - ); - assert_sql_count_choices(&choices_of(&inventory, 0)); -} - -/// A `HAVING` over an aggregate value, as a `Filter` above or as the -/// aggregate's own `having`, stays above the output and keeps executable candidates. -#[test] -fn value_having_stays_above_output_and_keeps_candidates() { - let value = || equals(1, ScalarValue::Int64(10)); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(having_filter(grouped_count(None), value())), - QueryRoot::Operator(grouped_count(Some(value()))), - ])) - .unwrap(); - let targets = &inventory.targets; - assert_eq!(targets.len(), 2); - for (index, target) in targets.iter().enumerate() { - let placements: Vec<_> = target - .having_filters - .iter() - .map(|entry| entry.placement) - .collect(); - assert_eq!(placements, vec![HavingPlacement::AboveOutput]); - assert_sql_count_choices(&choices_of(&inventory, index)); - } -} - -/// `avg`'s one exact option is sum and count states when its input is never -/// null, and plain `Exact` when that rewrite is pruned. -#[test] -fn avg_rewrites_to_sum_and_count_states() { - let avg = || vec![AggIntent::Avg { col: None }]; - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(aggregate_over(scan(false), avg(), vec![], None)), - QueryRoot::Operator(aggregate_over(scan(true), avg(), vec![], None)), - ])) - .unwrap(); - let targets = &inventory.targets; - assert_eq!( - targets[0].ways_to_compute, - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum, - ExactKind::Count - ])] - ); - assert_eq!( - targets[1].ways_to_compute, - vec![HowToComputeAggregateMeasure::Exact] - ); - assert_eq!( - inventory.pruned_of_target(1, false), - vec![( - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum, ExactKind::Count]), - PruneReason::NullableAvgInput - )] - ); -} - -/// A key-only `HAVING` above a shared aggregate stays above the output when -/// another consumer reads the aggregate without it. -#[test] -fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { - let node = grouped_count(None); - let key = equals(0, ScalarValue::Utf8("a".into())); - let filtered = having_filter(node.clone(), key.clone()); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(filtered.clone()), - QueryRoot::Operator(node.clone()), - ])) - .unwrap() - .targets; - assert_eq!(targets.len(), 1); - assert_eq!( - targets[0].having_filters, - vec![HavingEntry { - filter: Some(filtered), - predicate: key.clone(), - placement: HavingPlacement::AboveOutput, - }] - ); - - // Two consumers applying the same key-only predicate push it below. - let first = having_filter(node.clone(), key.clone()); - let second = having_filter(node, key); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(first), - QueryRoot::Operator(second), - ])) - .unwrap() - .targets; - assert!(targets[0] - .having_filters - .iter() - .all(|entry| entry.placement == HavingPlacement::BelowAggregate)); - // The two equal `Filter`s are one memo group, so one HAVING entry. - assert_eq!(targets[0].having_filters.len(), 1); -} - -/// A scalar reference to a grouped aggregate (`Filter(scan, EXISTS G)` as a -/// second root) is a consumer that reads it without the `HAVING`, so the -/// first root's key-only `HAVING` stays above the output. -#[test] -fn a_scalar_reference_consumer_keeps_a_key_only_having_above() { - let node = grouped_count(None); - let key = equals(0, ScalarValue::Utf8("a".into())); - let filtered = having_filter(node.clone(), key.clone()); - let exists = having_filter( - scan(false), - Predicate(ScalarExpr::Exists { - subquery: node.clone(), - negated: false, - }), - ); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(filtered.clone()), - QueryRoot::Operator(exists), - ])) - .unwrap(); - let targets = &inventory.targets; - let target = targets - .iter() - .find(|target| **inventory.aggregate(target) == *node) - .expect("the grouped count's target"); - assert_eq!( - target.having_filters, - vec![HavingEntry { - filter: Some(filtered), - predicate: key, - placement: HavingPlacement::AboveOutput, - }] - ); -} - -/// Of two stacked `Filter`s above an aggregate, only the adjacent one is -/// its `HAVING`; the outer one filters the inner `Filter`'s output. -#[test] -fn only_the_adjacent_of_stacked_filters_is_having() { - let node = grouped_count(None); - let inner_key = equals(0, ScalarValue::Utf8("a".into())); - let inner = having_filter(node.clone(), inner_key.clone()); - let outer = having_filter(inner.clone(), equals(0, ScalarValue::Utf8("b".into()))); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(outer), - ])) - .unwrap() - .targets; - assert_eq!(targets.len(), 1); - assert_eq!( - targets[0].having_filters, - vec![HavingEntry { - filter: Some(inner), - predicate: inner_key, - placement: HavingPlacement::BelowAggregate, - }] - ); -} - -/// A global aggregate's `HAVING` always stays above the output, even a -/// constant one: the one output row exists with no input rows. -#[test] -fn global_aggregate_having_stays_above() { - let count = || { - vec![AggIntent::Count { - accuracy: approximate(), - }] - }; - let never = Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(having_filter( - aggregate_over(scan(false), count(), vec![], None), - never.clone(), - )), - QueryRoot::Operator(aggregate_over(scan(false), count(), vec![], Some(never))), - ])) - .unwrap() - .targets; - assert_eq!(targets.len(), 2); - for target in &targets { - let placements: Vec<_> = target - .having_filters - .iter() - .map(|entry| entry.placement) - .collect(); - assert_eq!(placements, vec![HavingPlacement::AboveOutput]); - } -} - -/// #509 Example 1 Q2 under the default exact target: the top-k's one option is plain -/// `Exact`, its heap summaries pruned `RequiresExact`; `sum_over_time` gets -/// its exact Sum accumulator. -#[test] -fn example_1_exact_q2_prunes_topk_heaps() { - let roots = exact_promql("topk by (job) (10, sum_over_time(http_requests_total[1m]))"); - let candidates = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); - let found: Vec<_> = candidates - .targets - .iter() - .enumerate() - .map(|(index, _)| choices_of(&candidates, index)) - .collect(); - let topk = found - .iter() - .find(|choices| choices.candidates == [HowToComputeAggregateMeasure::Exact]) - .expect("top-k target"); - assert_eq!( - topk.pruned, - vec![ - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - PruneReason::RequiresExact - ), - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - PruneReason::RequiresExact - ), - ] - ); - assert!(found.iter().any(|choices| choices.candidates - == [HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum - ])] - && choices.pruned.is_empty())); - assert_eq!(found.len(), 2, "{found:#?}"); -} - -/// The `HAVING` entries of the target whose measure satisfies `is`. -fn having_of( - inventory: &crate::pass1::per_computation_candidates::PerComputationCandidates, - is: impl Fn(&AggIntent) -> bool, -) -> Vec { - let target = inventory - .targets - .iter() - .find(|target| match inventory.aggregate(target).non_asap() { - Some(NonASAPOp::Aggregate { measures, .. }) => is(&measures[target.measure_index]), - _ => false, - }) - .expect("target"); - target.having_filters.clone() -} - -/// `Column(column) > ScalarSubquery(subquery)`. -fn above_scalar(column: usize, subquery: Rc) -> Predicate { - Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(column)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::ScalarSubquery(subquery)), - semantics: ExprSemantics::Sql, - }) -} - -/// A `Filter` that reads an aggregate only from its predicate is not that -/// aggregate's `HAVING`; one that also filters the aggregate gives one entry. -#[test] -fn scalar_reference_filter_is_not_having() { - let count = || { - vec![AggIntent::Count { - accuracy: approximate(), - }] - }; - let agg = aggregate_over(scan(false), count(), vec![], None); - let root = having_filter(scan(false), above_scalar(1, agg.clone())); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ])) - .unwrap(); - let targets = &inventory.targets; - assert_eq!(targets.len(), 1); - assert_eq!(**inventory.aggregate(&targets[0]), *agg); - assert!( - targets[0].having_filters.is_empty(), - "{:?}", - targets[0].having_filters - ); - - let agg = aggregate_over(scan(false), count(), vec![], None); - let root = having_filter(agg.clone(), above_scalar(0, agg.clone())); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root.clone()), - ])) - .unwrap() - .targets; - assert_eq!(targets.len(), 1); - assert_eq!( - targets[0].having_filters, - vec![HavingEntry { - filter: Some(root), - predicate: above_scalar(0, agg), - placement: HavingPlacement::AboveOutput, - }] - ); -} - -/// PromQL `scalar(...)` operands are scalar references, not `HAVING`. -#[test] -fn promql_scalar_operand_is_not_having() { - let roots = exact_promql("http_requests_total > scalar(sum(http_requests_total))"); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap() - .targets; - assert!(!targets.is_empty()); - for target in &targets { - assert!( - target.having_filters.is_empty(), - "{:?}", - target.having_filters - ); - } - - let roots = exact_promql("sum by (job) (x) > scalar(count(x))"); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); - assert!(having_of(&inventory, |intent| matches!( - intent, - AggIntent::Count { .. } - )) - .is_empty()); - let sum = having_of(&inventory, |intent| matches!(intent, AggIntent::Sum { .. })); - assert_eq!(sum.len(), 1, "{sum:?}"); - assert_eq!(sum[0].placement, HavingPlacement::AboveOutput); -} - -/// A non-root, non-`Filter` consumer keeps a key-only `HAVING` above. -#[test] -fn having_shared_with_a_project_consumer_stays_above() { - let node = grouped_count(None); - let key = equals(0, ScalarValue::Utf8("a".into())); - let filtered = having_filter(node.clone(), key); - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: vec![asap_types::ir::ProjectItem { - alias: None, - expr: ScalarExpr::Column(0), - }], - qualifier: None, - child: node, - })) - .unwrap(); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(filtered), - QueryRoot::Operator(project), - ])) - .unwrap() - .targets; - assert_eq!(targets.len(), 1); - let placements: Vec<_> = targets[0] - .having_filters - .iter() - .map(|entry| entry.placement) - .collect(); - assert_eq!(placements, vec![HavingPlacement::AboveOutput]); -} - -/// `without(...)` and per-entity aggregates have no fixed key prefix, so a -/// `HAVING` over column 0 stays above. -#[test] -fn without_and_per_entity_having_stays_above() { - for reduction in [ - Reduction::Reduce( - asap_types::ir::operator::operator_properties::GroupKeys::without(vec![1]), - ), - Reduction::PerEntity, - ] { - let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction: reduction.clone(), - measures: vec![AggIntent::Count { - accuracy: approximate(), - }], - output_names: vec![], - filters: vec![], - having: None, - })) - .unwrap(); - let root = having_filter(node, equals(0, ScalarValue::Utf8("a".into()))); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ])) - .unwrap() - .targets; - assert_eq!(targets.len(), 1, "{reduction:?}"); - let placements: Vec<_> = targets[0] - .having_filters - .iter() - .map(|entry| entry.placement) - .collect(); - assert_eq!( - placements, - vec![HavingPlacement::AboveOutput], - "{reduction:?}" - ); - } -} - -/// A workload candidate replaces only chosen targets, derives the summary's -/// coverage from its subtree, and keeps unchosen plans as they are, now -/// with an exact guarantee. -#[test] -fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { - use crate::pass1::per_computation_candidates::compose_logical_candidate; - let producer = aggregate(AggIntent::Cardinality { - cols: vec![0], - accuracy: approximate(), - }); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(producer.clone()), - ])) - .unwrap(); - let exact = compose_logical_candidate(&inventory, &[0]).unwrap(); - let QueryRoot::Operator(kept) = &exact[0] else { - panic!("operator root expected") - }; - assert_eq!(kept.operator.kind_name(), producer.operator.kind_name()); - assert_eq!(kept.schema, producer.schema); - assert!(kept.guarantee.as_ref().is_some_and(|g| g.is_exact())); - - let hll = compose_logical_candidate(&inventory, &[1]).unwrap(); - let QueryRoot::Operator(estimate) = &hll[0] else { - panic!("operator root expected") - }; - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = estimate.asap() else { - panic!("summary evaluation expected") - }; - // The state takes every row of its source: one unrestricted selection box. - let coverage = summary_input.coverage().unwrap(); - assert!(OperatorNode::reachable(coverage.definition()).iter().any(|node| matches!( - node.non_asap(), - Some(NonASAPOp::Scan { source: Source::Table { table_ref }, .. }) if table_ref == "flows" - ))); - assert_eq!(coverage.selection().len(), 1); - assert!( - coverage.selection()[0].columns().is_empty() - && coverage.selection()[0].relative_time().is_none() - ); - hll[0].validate_structure().unwrap(); - assert!(compose_logical_candidate(&inventory, &[99]).is_err()); -} - -/// The one root of the workload candidate composed from `roots` and `choice`. -fn composed(roots: Vec>, choice: &[usize]) -> Vec> { - use crate::pass1::per_computation_candidates::compose_logical_candidate; - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups( - &roots - .into_iter() - .map(QueryRoot::Operator) - .collect::>(), - )) - .unwrap(); - compose_logical_candidate(&inventory, choice) - .unwrap() - .into_iter() - .map(|root| { - root.validate_structure().unwrap(); - match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator root expected"), - } - }) - .collect() -} -/// Output columns as (name, type, nullable). -fn columns(node: &OperatorNode) -> Vec<(String, FieldDataType, bool)> { - node.schema - .fields - .iter() - .map(|field| (field.name.clone(), field.dtype.clone(), field.nullable)) - .collect() -} -// Quantile readouts restore the aggregate's output names with an identity projection. -fn without_output_aliases(node: &OperatorNode) -> &OperatorNode { - if let Some(NonASAPOp::Project { cols, child, .. }) = node.non_asap() { - assert!(cols - .iter() - .enumerate() - .all(|(index, item)| item.expr == ScalarExpr::Column(index))); - without_output_aliases(child) - } else { - node - } -} -fn summary_agg(node: &OperatorNode) -> &ASAPOp { - let node = without_output_aliases(node); - match node.asap() { - Some(ASAPOp::SummaryEstimate { summary_input, .. }) => summary_agg(summary_input), - Some(ASAPOp::FinalizeExactAccumulator { child }) => summary_agg(child), - Some(agg @ ASAPOp::SummaryAgg { .. }) => agg, - _ => panic!("summary evaluation expected, got {node:#?}"), - } -} - -/// A summary is sized when the workload candidate is composed, for its -/// measure's accuracy requirement. -#[test] -fn composed_summary_is_sized_for_its_measure() { - use asap_accuracy::test_support::default_size_params; - let intent = quantile(); - let [root] = <[_; 1]>::try_from(composed(vec![aggregate(intent.clone())], &[1])).unwrap(); - let ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } = summary_agg(&root) - else { - panic!("sketch state expected") - }; - assert_eq!(*kind.algorithm(), SketchAlgorithm::Kll); - assert_eq!( - kind.params(), - &default_size_params(SketchAlgorithm::Kll, &intent, 0.05, 0.01) - ); -} - -/// A measure filter becomes the state's filter; a key-only `HAVING` applied -/// by every consumer moves below the state; a value `HAVING` stays a `Filter` -/// above the estimate. -#[test] -fn composed_filters_and_having() { - let kll = 1; - let filtered = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction: Reduction::by(vec![0]), - measures: vec![quantile()], - output_names: vec![], - filters: vec![Some(always())], - having: None, - })) - .unwrap(); - let [root] = <[_; 1]>::try_from(composed(vec![filtered.clone()], &[kll])).unwrap(); - assert!( - matches!(summary_agg(&root), ASAPOp::SummaryAgg { filter: Some(f), child, .. } - if *f == always() && child.non_asap().is_some_and(|op| matches!(op, NonASAPOp::Scan { .. }))) - ); - assert_eq!(columns(&root)[0], columns(&filtered)[0]); - - let key = equals(0, ScalarValue::Utf8("a".into())); - let [root] = <[_; 1]>::try_from(composed( - vec![having_filter(grouped_quantile(None), key.clone())], - &[kll], - )) - .unwrap(); - let ASAPOp::SummaryAgg { child, filter, .. } = summary_agg(&root) else { - unreachable!() - }; - assert_eq!(*filter, None); - assert!(matches!(child.non_asap(), Some(NonASAPOp::Filter { pred, .. }) if *pred == key)); - - let value = || equals(1, ScalarValue::Int64(10)); - for root in composed( - vec![ - having_filter(grouped_quantile(None), value()), - grouped_quantile(Some(value())), - ], - &[kll, kll], - ) { - let Some(NonASAPOp::Filter { pred, child }) = root.non_asap() else { - panic!("value HAVING above the estimate expected, got {root:#?}") - }; - assert_eq!(*pred, value()); - assert!(matches!( - without_output_aliases(child).asap(), - Some(ASAPOp::SummaryEstimate { .. }) - )); - } -} - -/// SQL `avg` joins its finalized `sum` and `count` accumulator states on the -/// group keys and divides them. -#[test] -fn composed_avg_divides_sum_and_count_states() { - let avg = aggregate_over( - scan(false), - vec![AggIntent::Avg { col: Some(1) }], - vec![], - None, - ); - let [root] = <[_; 1]>::try_from(composed(vec![avg.clone()], &[0])).unwrap(); - // The quotient is renamed to the aggregate's column. - assert_eq!(columns(&root), columns(&avg)); - let join = OperatorNode::reachable(&root) - .into_iter() - .find(|node| matches!(node.non_asap(), Some(NonASAPOp::Join { .. }))) - .unwrap_or_else(|| panic!("sum joined with count expected, got {root:#?}")); - let Some(NonASAPOp::Join { left, right, .. }) = join.non_asap() else { - unreachable!() - }; - for (side, kind) in [(left, ExactKind::Sum), (right, ExactKind::Count)] { - assert!(matches!( - summary_agg(side), - ASAPOp::SummaryAgg { family: FieldDataType::ExactAggregate(found, _), .. } if *found == kind - )); - } -} - -/// Each measure of a multi-measure aggregate is substituted on its own and the -/// measures are joined back on the group keys, with the aggregate's columns. -#[test] -fn composed_multi_measure_aggregate_joins_its_measures() { - for reduction in [Reduction::by(vec![0]), Reduction::by(vec![])] { - let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction, - measures: vec![ - AggIntent::Sum { col: Some(1) }, - quantile(), - AggIntent::StdDev { - col: Some(1), - population: true, - }, - ], - output_names: vec![], - filters: vec![], - having: None, - })) - .unwrap(); - let [root] = <[_; 1]>::try_from(composed(vec![node.clone()], &[0, 1])).unwrap(); - assert!(matches!(root.non_asap(), Some(NonASAPOp::Project { .. }))); - let reached = OperatorNode::reachable(&root); - let joins = reached - .iter() - .filter(|node| matches!(node.non_asap(), Some(NonASAPOp::Join { .. }))) - .count(); - assert_eq!(joins, 2); - assert!(reached - .iter() - .any(|node| matches!(node.asap(), Some(ASAPOp::SummaryEstimate { .. })))); - let names = |node: &OperatorNode| -> Vec<_> { - columns(node) - .into_iter() - .map(|(name, dtype, _)| (name, dtype)) - .collect() - }; - assert_eq!(names(&root), names(&node)); - } -} - -/// A measure left exact beside a sketched one keeps its output name: the -/// split exact aggregate names its one measure as the target named it, so the -/// executor finds one name per measure. -#[test] -fn composed_mixed_exact_and_sketch_measures_keep_output_names() { - let quantile = |q| AggIntent::Quantile { - col: Some(1), - q, - accuracy: approximate(), - }; - let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction: Reduction::by(vec![]), - measures: vec![quantile(0.5), quantile(0.9)], - output_names: vec!["p50".into(), "p90".into()], - filters: vec![], - having: None, - })) - .unwrap(); - assert_eq!( - candidates_for(quantile(0.5))[0], - HowToComputeAggregateMeasure::Exact - ); - // p50 exact, p90 from the first sketch family. - let [root] = <[_; 1]>::try_from(composed(vec![node.clone()], &[0, 1])).unwrap(); - let exact_parts: Vec<_> = OperatorNode::reachable(&root) - .into_iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Aggregate { - measures, - output_names, - .. - }) => Some((measures.len(), output_names.clone())), - _ => None, - }) - .collect(); - assert_eq!(exact_parts, vec![(1, vec!["p50".to_string()])]); - assert_eq!(columns(&root), columns(&node)); -} - -/// A summary estimate carries its sized family's bound; an exact accumulator -/// is exact; anything computed from an approximate value, a value `HAVING` -/// included, has no guarantee. -#[test] -fn composed_guarantees() { - let value = equals(1, ScalarValue::Int64(10)); - let roots = composed( - vec![ - aggregate(AggIntent::Cardinality { - cols: vec![0], - accuracy: approximate(), - }), - aggregate(AggIntent::Sum { col: Some(1) }), - having_filter(grouped_quantile(None), value), - ], - &[1, 0, 1], - ); - let guarantee = |node: &OperatorNode| node.guarantee.clone(); - let hll = guarantee(&roots[0]).expect("HLL bound"); - assert!(!hll.is_exact()); - assert!(guarantee(&roots[1]).is_some_and(|g| g.is_exact())); - let Some(NonASAPOp::Filter { child, .. }) = roots[2].non_asap() else { - panic!("value HAVING above the estimate expected") - }; - assert!(guarantee(child).is_some()); - assert!(guarantee(&roots[2]).is_none()); -} - -/// The aggregate's own value `having` stays a `Filter` above the output, and -/// a subquery in it reads the subquery's substitution. -#[test] -fn own_having_subquery_is_substituted() { - use crate::pass1::per_computation_candidates::compose_logical_candidate; - let inner = aggregate(quantile()); - let outer = grouped_quantile(Some(above_scalar(1, inner.clone()))); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(outer.clone()), - ])) - .unwrap(); - assert_eq!(inventory.targets.len(), 2); - let kll = 1; - let composed = compose_logical_candidate(&inventory, &[kll, kll]).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root expected") - }; - let Some(NonASAPOp::Filter { pred, child }) = root.non_asap() else { - panic!("own HAVING above the estimate expected, got {root:#?}") - }; - assert!(matches!( - without_output_aliases(child).asap(), - Some(ASAPOp::SummaryEstimate { .. }) - )); - let [subquery] = <[_; 1]>::try_from(pred.0.operator_refs()).unwrap(); - assert!( - matches!( - without_output_aliases(subquery).asap(), - Some(ASAPOp::SummaryEstimate { .. }) - ), - "{subquery:#?}" - ); - assert!(OperatorNode::reachable(root) - .iter() - .all(|node| !matches!(node.non_asap(), Some(NonASAPOp::Aggregate { .. })))); -} - -/// A per-series `avg` over PromQL rows that carry the series identity -/// (`$promql_series_identity` after the sample value) composes from its sum -/// and count states: the summed value is found by role, not as the last -/// column. -#[test] -fn per_series_avg_with_series_identity_composes_from_sum_and_count() { - use asap_types::ir::schema::schema_support::with_promql_series_identity; - let [QueryRoot::Operator(root)] = &exact_promql("avg_over_time(m[5m])")[..] else { - panic!("one operator root"); - }; - let root = with_promql_series_identity(root).unwrap(); - let [composed] = <[_; 1]>::try_from(composed(vec![root.clone()], &[0])).unwrap(); - assert_eq!(columns(&composed), columns(&root)); -} - -/// Exact per-series `rate` keeps every label and retypes the value column to the Rate state. -#[test] -fn per_series_rate_keeps_labels_and_retypes_its_state() { - use asap_types::ir::schema::ExactParams; - let [QueryRoot::Operator(root)] = &exact_promql("rate(m{job=\"api\"}[5m])")[..] else { - panic!("one operator root"); - }; - let [out] = <[_; 1]>::try_from(composed(vec![root.clone()], &[0])).unwrap(); - let rate = FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate); - let ASAPOp::SummaryAgg { - family, reduction, .. - } = summary_agg(&out) - else { - unreachable!() - }; - assert_eq!(*family, rate); - assert_eq!(*reduction, Reduction::PerEntity); - let state = OperatorNode::reachable(&out) - .into_iter() - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .unwrap(); - // The state replaces the sample value in place, named `state`; the - // finalized output has the rate's own columns. - let label = Field::plain("job", DataType::Utf8, true); - assert_eq!( - columns(&state), - [ - ( - "ts".into(), - FieldDataType::Plain(DataType::Timestamp), - false - ), - ("state".into(), rate, false), - (label.name, label.dtype, label.nullable), - ] - ); - assert_eq!(state.schema.time_index, Some(0)); - assert_eq!(out.schema, root.schema); -} - -/// Exact grouped `avg` composes from sum and count with the aggregate's whole schema. -#[test] -fn grouped_avg_composition_preserves_the_whole_schema() { - let [QueryRoot::Operator(root)] = &exact_promql("avg by (job) (m)")[..] else { - panic!("one operator root"); - }; - let [out] = <[_; 1]>::try_from(composed(vec![root.clone()], &[0])).unwrap(); - assert!(OperatorNode::reachable(&out) - .iter() - .any(|node| matches!(node.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))); - assert_eq!(out.schema, root.schema); -} - -// Structurally equal aggregates are one memo group with one target, and its -// HAVING placement is decided over every consumer of the group: an unfiltered -// consumer keeps the key-only HAVING above the output for the filtered ones. -#[test] -fn memo_cached_rules_preserve_occurrence_specific_having() { - let shared = grouped_count(None); - let other = grouped_count(None); - let key = equals(0, ScalarValue::Utf8("a".into())); - let roots = vec![ - QueryRoot::Operator(having_filter(shared.clone(), key.clone())), - QueryRoot::Operator(shared), - QueryRoot::Operator(having_filter(other, key)), - ]; - let memo = generate_dag_of_logical_memo_groups(&roots); - let cached = generate_per_computation_candidates(&memo).unwrap(); - assert_eq!(cached.targets.len(), 1); - // Roots 0 and 2 apply the same `Filter`, one memo group: one entry. - let [entry] = &cached.targets[0].having_filters[..] else { - panic!("one HAVING entry: {:?}", cached.targets[0].having_filters); - }; - assert_eq!(entry.placement, HavingPlacement::AboveOutput); -} diff --git a/crates/logical-asap-optimization/src/pass1/realization.rs b/crates/logical-asap-optimization/src/pass1/realization.rs index d5dc1f09c..5628f709f 100644 --- a/crates/logical-asap-optimization/src/pass1/realization.rs +++ b/crates/logical-asap-optimization/src/pass1/realization.rs @@ -1,21 +1,20 @@ -//! Summary realization helpers shared by Pass 1 (`per_computation_candidates`) +//! Summary realization helpers shared by Pass 1 (`computation_targets`) //! and Pass 2 (`asap_aware_cse`): which sketch kinds can serve an intent, //! the column/tuple a summary consumes, the column a top-k ranks, and the //! keyed-additive rule for a heap sketch that reads a larger sub-DAG. use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::operator::Reduction; -use asap_types::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; +use asap_types::ir::scalar::column_resolution::resolve_column_ref; use asap_types::ir::scalar::expr_ir::ColumnRef; -use asap_types::ir::schema::topk::ranked_column; -use asap_types::ir::schema::ColumnId; -use asap_types::ir::schema::{ - EntityIdentity, FamilyCapability, Field, FieldDataType, NonNegativeWeightProof, Schema, - SketchAlgorithm, SummaryInputExpr, SummaryUpdate, WeightDomain, -}; +use asap_types::ir::schema::{Field, FieldDataType}; use asap_types::ir::{NonASAPOp, OperatorNode}; #[cfg(test)] use asap_types::ir::{Operator, ScalarExpr}; +use asap_types::summary_type::{ + EntityIdentity, NonNegativeWeightProof, SketchAlgorithm, SummaryInputExpr, + SummaryTypeCapability, SummaryUpdate, WeightDomain, +}; use std::rc::Rc; use asap_accuracy::sizing::sketch_statistic; @@ -35,19 +34,21 @@ const FAMILY_PREFERENCE: &[SketchAlgorithm] = &[ SketchAlgorithm::UnivMon, ]; -/// The sketch kinds that can serve an intent, most-preferred first: the -/// families whose family-capability table row answers the intent's -/// statistic, in preference order, less UnivMon for a distinct-tuple +/// The sketch algorithms that can serve an intent, most-preferred first: +/// those whose capability table row answers the intent's statistic, in +/// preference order, less UnivMon for a distinct-tuple /// count. Empty for an intent no sketch /// statistic serves. pub(crate) fn summary_candidates(intent: &AggIntent) -> Vec { - candidates_by(intent, FamilyCapability::of_per_instance_sketch) + candidates_by(intent, |algorithm| { + SummaryTypeCapability::of(&FieldDataType::Sketch(*algorithm)) + }) } -/// [`summary_candidates`] with each family's table row read from `capability`. +/// [`summary_candidates`] with each algorithm's table row read from `capability`. fn candidates_by( intent: &AggIntent, - capability: impl Fn(&SketchAlgorithm) -> FamilyCapability, + capability: impl Fn(&SketchAlgorithm) -> SummaryTypeCapability, ) -> Vec { let Some(statistic) = sketch_statistic(intent) else { return Vec::new(); @@ -66,60 +67,6 @@ fn candidates_by( .collect() } -/// The column fed into a *single-column* summary: the intent's leading -/// positional input resolved to a name against the child schema, or the PromQL -/// sample value when it reads none. Callers are responsible for only reaching -/// here with a one-column intent — [`summarised_input`] is the general form. -pub(crate) fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> ColumnRef { - match intent - .input_cols() - .first() - .and_then(|id| child_schema.fields.get(*id)) - { - Some(c) => c.column_ref(), - None => ColumnRef::SampleValue, - } -} - -/// [`ranked_column`] as a reference: the sample value when it is that -/// column, else the column by name. -pub(crate) fn ranked_value(child: &OperatorNode) -> Result<(ColumnId, ColumnRef), ResolveError> { - let ranked = ranked_column(child)?; - let column = match resolve_column_ref(&ColumnRef::SampleValue, &child.schema) { - Ok(value) if value == ranked => ColumnRef::SampleValue, - _ => child.schema.fields[ranked].column_ref(), - }; - Ok((ranked, column)) -} - -/// What the summary consumes per input row. An intent that reads one column (or -/// none) feeds that column; `COUNT(DISTINCT a, b)` feeds the whole tuple as one -/// item, so the distinct-count sketch hashes `(a, b)` rather than `a` — the -/// difference between tuple cardinality and single-column cardinality. -/// -/// A tuple leg outside the child schema is an error rather than -/// [`summarised_column`]'s sample-value fallback: a leg has no sample-value -/// reading, and silently dropping one would under-count: `None` then. -pub(crate) fn summarised_input( - intent: &AggIntent, - child_schema: &Schema, -) -> Option { - let cols = intent.input_cols(); - if cols.len() < 2 { - return Some(SummaryInputExpr::Column(summarised_column( - intent, - child_schema, - ))); - } - let legs = cols - .iter() - .map(|id| child_schema.fields.get(*id).map(Field::column_ref)) - .collect::>>()?; - Some(SummaryInputExpr::Tuple( - legs.into_iter().map(SummaryInputExpr::Column).collect(), - )) -} - /// The physical input consumed by one summary realization. Most summaries /// consume the logical aggregate's immediate child and summarize its declared /// input value. Composite realizations can instead consume a larger @@ -143,10 +90,10 @@ pub(crate) fn realize_keyed_additive_summary_input( if !matches!(intent, AggIntent::TopK { .. }) { return None; } - let FieldDataType::Sketch(kind, _) = family else { + let FieldDataType::Sketch(kind) = family else { return None; }; - let heap_algorithm = kind.algorithm(); + let heap_algorithm = kind; if !matches!( heap_algorithm, SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap @@ -197,8 +144,8 @@ pub(crate) fn realize_keyed_additive_summary_input( let samples = summed_per_series_samples(child); // A raw data sample's count or a count's row weighs one. let counted = match &samples { - Some((_, inner)) => matches!(inner, AggIntent::Count { .. }), - None => matches!(measures.as_slice(), [AggIntent::Count { .. }]), + Some((_, inner)) => matches!(inner, AggIntent::Count), + None => matches!(measures.as_slice(), [AggIntent::Count]), }; let weight = match measures.as_slice() { _ if counted => SummaryInputExpr::Constant(1.0), @@ -319,8 +266,7 @@ fn summed_per_series_samples(grouped: &OperatorNode) -> Option<(Rc else { return None; }; - let [inner @ (AggIntent::Sum { col: None } | AggIntent::Count { .. })] = inner.as_slice() - else { + let [inner @ (AggIntent::Sum { col: None } | AggIntent::Count)] = inner.as_slice() else { return None; }; if by.is_without() @@ -351,9 +297,7 @@ fn summed_per_series_samples(grouped: &OperatorNode) -> Option<(Rc mod tests { use super::*; use crate::test_support::metric_scan; - use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; - use asap_types::ir::schema::{GroupingStrategy, SketchKind, SketchParams}; use asap_types::ir::{ExprSemantics, Predicate}; /// A heap sketch consumes every raw row, so the keyed input rule refuses @@ -363,9 +307,7 @@ mod tests { let count_by_job = |filter: Option| { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }], + measures: vec![AggIntent::Count], output_names: vec![], filters: vec![filter], having: None, @@ -373,23 +315,9 @@ mod tests { })) .unwrap() }; - let intent = AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::max_error(0.01), - }; - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 4, - heap_size: 5, - }, - ), - GroupingStrategy::default(), - ); - let output = - Reduction::Reduce(asap_types::ir::operator::operator_properties::GroupKeys::none()); + let intent = AggIntent::TopK { k: 5 }; + let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); + let output = Reduction::Reduce(asap_types::ir::operator::arguments::GroupKeys::none()); assert!(realize_keyed_additive_summary_input( &intent, &family, @@ -412,85 +340,28 @@ mod tests { .is_none()); } - /// A summary reads the intent's own input columns: one column by name, - /// none as the sample value, several as one hashed tuple. - #[test] - fn summarised_input_reads_the_intents_columns() { - // `metric_scan(&["job"])` → columns [ts=0, value=1, job=2]. - let scan = metric_scan(&["job"]); - let column = |name: &str| SummaryInputExpr::Column(ColumnRef::Named(name.into())); - let cases = [ - (vec![2], column("job")), - (vec![1], column("value")), - (vec![], SummaryInputExpr::Column(ColumnRef::SampleValue)), - ( - vec![1, 2], - SummaryInputExpr::Tuple(vec![column("value"), column("job")]), - ), - ]; - for (cols, want) in cases { - let intent = AggIntent::Cardinality { - cols: cols.clone(), - accuracy: AccuracyTarget::max_error(0.01), - }; - assert_eq!( - summarised_input(&intent, &scan.schema), - Some(want), - "wrong summarised input for cols={cols:?}" - ); - } - } - /// Each approximable intent is offered the families it was offered when /// the list was written by hand, in the same order; a non-sketch intent /// is offered none. #[test] fn summary_candidates_keep_the_hand_written_lists() { use SketchAlgorithm as S; - let accuracy = AccuracyTarget::max_error(0.01); - let cardinality = |cols: Vec| AggIntent::Cardinality { - cols, - accuracy: accuracy.clone(), - }; + let cardinality = |cols: Vec| AggIntent::Cardinality { cols }; let cases = [ ( - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: accuracy.clone(), - }, + AggIntent::Quantile { col: None, q: 0.99 }, vec![S::Kll, S::DDSketch], ), (cardinality(vec![]), vec![S::Hll, S::Kmv, S::UnivMon]), (cardinality(vec![1]), vec![S::Hll, S::Kmv, S::UnivMon]), (cardinality(vec![1, 2]), vec![S::Hll, S::Kmv]), + (AggIntent::FrequencyL2 { col: None }, vec![S::UnivMon]), + (AggIntent::FrequencyEntropy { col: None }, vec![S::UnivMon]), ( - AggIntent::FrequencyL2 { - col: None, - accuracy: accuracy.clone(), - }, - vec![S::UnivMon], - ), - ( - AggIntent::FrequencyEntropy { - col: None, - accuracy: accuracy.clone(), - }, - vec![S::UnivMon], - ), - ( - AggIntent::TopK { - k: 10, - accuracy: accuracy.clone(), - }, + AggIntent::TopK { k: 10 }, vec![S::CmsWithHeap, S::CountSketchWithHeap], ), - ( - AggIntent::Count { - accuracy: accuracy.clone(), - }, - vec![S::Cms, S::UnivMon], - ), + (AggIntent::Count, vec![S::Cms, S::UnivMon]), (AggIntent::Sum { col: None }, vec![]), ]; for (intent, want) in cases { @@ -504,15 +375,12 @@ mod tests { fn a_dropped_readout_drops_the_family() { // Count-Min-with-heap answered from Count-Min's row: no top-k readout. let heap_without_top_k = |family: &SketchAlgorithm| { - FamilyCapability::of_per_instance_sketch(match family { - SketchAlgorithm::CmsWithHeap => &SketchAlgorithm::Cms, - other => other, - }) - }; - let top_k = AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::max_error(0.01), + SummaryTypeCapability::of(&FieldDataType::Sketch(match family { + SketchAlgorithm::CmsWithHeap => SketchAlgorithm::Cms, + other => *other, + })) }; + let top_k = AggIntent::TopK { k: 10 }; assert_eq!( candidates_by(&top_k, heap_without_top_k), vec![SketchAlgorithm::CountSketchWithHeap] diff --git a/crates/logical-asap-optimization/src/pass2.rs b/crates/logical-asap-optimization/src/pass2.rs index 2607e1edf..a51c6e902 100644 --- a/crates/logical-asap-optimization/src/pass2.rs +++ b/crates/logical-asap-optimization/src/pass2.rs @@ -1,11 +1,62 @@ -//! Stage 1, Pass 2: ASAP-aware sharing over the Pass 1 candidates, through the -//! logical memo and its rules. - -/// Stage 1, Pass 2: ASAP-aware common-subexpression elimination (the -/// identical-expression rule). -pub(crate) mod asap_aware_cse; -pub(crate) mod finer_aggregate; -/// Stage 1, Pass 2's entry, producing [`asap_types::stages::LogicalASAPMemoCandidates`]. -pub(crate) mod logical_memo_candidates; -/// The logical memo: candidates, rules and enumeration over Stage 0's groups. +//! Stage 1, Pass 2: the rules over groups and the logical edges. Its entry, +//! [`generate_logical_asap_memo_candidates`], extends the Pass 1 memo by the +//! rules over groups and adds the logical edges +//! ([`asap_types::stages::LogicalASAPMemoCandidates`]), which Stage 2 expands. + +/// The rules over groups. pub(crate) mod memo; + +use asap_accuracy::AccuracyEvidence; +use asap_types::stages::{LogicalASAPMemoCandidates, Memo}; +use asap_types::workload::QueryWorkloadEntry; +use memo::rules::{finer_aggregate, rollup, scan_labels, summary_inputs, topk, Pass2RuleError}; +use memo::window_candidates::add_local_window_composition; + +/// Stage 1, Pass 2: rule 1 ([`pass2_local_window_composition`]), then rules +/// 2–7 and the logical edges ([`pass2_after_local_window_composition`]). +/// `entries[i]` is query `i`'s workload entry; `evidence` is the +/// deployment's measured accuracy, which each edge's $\alpha^{\log}$ reads +/// before the theoretical guarantee. +pub fn generate_logical_asap_memo_candidates( + pass1: Memo, + entries: &[QueryWorkloadEntry], + evidence: &dyn AccuracyEvidence, +) -> Result { + pass2_after_local_window_composition(pass2_local_window_composition(pass1, entries)?, evidence) +} + +/// Pass 2's rule 1 alone: Pass 1's memo with the tumbling-window layouts +/// of each summary state group (`asap_planner` prints this step). +pub fn pass2_local_window_composition( + pass1: Memo, + entries: &[QueryWorkloadEntry], +) -> Result { + let mut memo = pass1; + // 1. Local window composition. + add_local_window_composition(&mut memo, entries)?; + Ok(memo) +} + +/// The rest of Pass 2 on [`pass2_local_window_composition`]'s memo: rules +/// 2–7, then the readouts' required accuracy and the logical edges, with +/// $\alpha^{\log}$ from `evidence` or the theoretical guarantee +/// (`asap_accuracy::producer_satisfies_logical_requirements`). +pub fn pass2_after_local_window_composition( + mut memo: Memo, + evidence: &dyn AccuracyEvidence, +) -> Result { + // 2. Scan widening. + scan_labels::add_scan_label_candidates(&mut memo)?; + // 3. Summary inputs. + summary_inputs::factor_summary_inputs(&mut memo)?; + // 4. Top-k reuse. + topk::add_topk_candidates(&mut memo)?; + // 5. Finer aggregate. + finer_aggregate::add_finer_aggregate_candidates(&mut memo)?; + // 6. Rollup. + rollup::add_rollup_candidates(&mut memo)?; + // 7. Window rollup. + rollup::add_window_rollup_candidates(&mut memo)?; + let _ = (memo, evidence); + todo!("#1207 PR 4: readout requirements, dead-candidate elimination and the logical edges") +} diff --git a/crates/logical-asap-optimization/src/pass2/asap_aware_cse.rs b/crates/logical-asap-optimization/src/pass2/asap_aware_cse.rs deleted file mode 100644 index 991e96f40..000000000 --- a/crates/logical-asap-optimization/src/pass2/asap_aware_cse.rs +++ /dev/null @@ -1,3557 +0,0 @@ -//! Stage 1 (logical ASAP-aware optimization), Pass 2: ASAP-aware -//! common-subexpression elimination, as defined in -//! `docs/design_docs/proposals/planner-layering.md` (Algorithm 2). -//! -//! The memo rules (`pass2::memo::rules`) are built on these helpers: the -//! window-composition rule for tumbling windows, and two reuse rules (#680), -//! rollup and exact top-k reuse. Identical expressions are shared by memo -//! group identity, and summary builds of one family at different sizes by -//! the memo's shared-producer rule (`memo::rules::shared_producer`); both -//! rules' whole-workload forms are gone. -//! -//! Scans of one metric that project different labels read the same rows. -//! [`with_union_scan_labels`] widens them to the union of their labels, so -//! interning finds them identical (`memo::rules::scan_labels`). -//! -//! # Window-composition rule -//! -//! A query repeating every `E` over a window `W` can build its summary as -//! back-to-back tumbling windows of width `w = gcd(W, E)` (Algorithm 2's -//! `Tumbling(L)` for one query), merged before each read. A *tumbling window -//! candidate* ([`tumbling_window_candidates`]) is a `SummaryAgg` over a range -//! `TimeRange(W)` whose -//! family merges in the family-capability table ([`FamilyCapability`]): -//! exact accumulators included, and also families whose merged readouts -//! are only approximate (heap top-k), whose readers then carry no -//! guarantee and are left to plan selection's evidence. A family without a -//! merge (IRate) has no tumbling-window candidate. -//! [`window_composition_pass2_candidates`] returns one Pass 2 candidate per -//! non-empty subset of the tumbling-window candidates: the build of each in the subset -//! becomes a `SummaryMerge` over one [`ASAPOp::SummaryTumblingWindowChain`] of `W / w` tumbling windows, -//! a compact node whatever the tumbling-window count, with the build's family and -//! size (a mergeable family's merged state has one build's guarantee), and -//! every guarantee above is composed again. Nothing is materialized here: -//! the tumbling windows are rebuilt at each evaluation until Stage 2 keeps them. -//! -//! Nested windows share one chain: when a subset holds builds of one family -//! (at any size) over the same input below their ranges, read the same way, -//! with the same output column names, whose tumbling windows differ in width or count, -//! the subset also comes as a Pass 2 candidate in which each such group -//! reads one chain ([`SharedTumblingWindowChain`]) over the longest window, sized for -//! every member ([`size_for_all_readers`]). Two grids are offered: the -//! finest member's own tumbling windows, for the members whose widths it divides -//! ([`nested_window_groups`]), and, when that differs, the gcd of all the -//! members' widths, even finer than every member's own tumbling windows -//! ([`gcd_window_groups`]); Stage 3's pricing weighs the extra states and -//! merges of finer tumbling windows. Queries repeating at different intervals read the -//! same tumbling windows, each at its own cadence. Every window ends at the evaluation -//! time, so each shorter one is the chain's newest tumbling windows: its -//! `SummaryMerge` merges only those (`newest_tumbling_windows`), and has the guarantee -//! of one build over its own window. A build grouped by fewer keys also -//! reads the chain of a finer grouping over a window at least its own -//! ([`rollup_window_groups`]): it merges its newest tumbling windows, then rolls the -//! groups up into its own, as the rollup rule does. Stage 2 never keeps a -//! chain read at different intervals across evaluations from query time -//! (each reader would need one new tumbling window per evaluation). Long-window segments, sliding -//! windows and Exponential Histograms are not implemented. -//! -//! # Rollup rule -//! -//! A coarse grouped computation can be answered from a finer one the -//! candidate already computes over the same rows: `sum by (job)` from -//! `sum by (job, instance)`. The rule never introduces a finer grouping -//! nobody asked for. A *rollup candidate* ([`rollup_candidates`]) is a -//! coarse build and a finer build of the same family over the same rows (a -//! PromQL scan may project other labels), whose group keys include the -//! coarse ones: -//! - An exact accumulator whose family merges across disjoint groups with -//! an exact merged value ([`rollup_kind`]: sums, counts, minima and -//! maxima; counter states merge only in time). The coarse build becomes -//! an exact accumulator over its coarsest finer build's finalized values, -//! grouped by the coarse keys (a count sums the finer counts), so the -//! value stays exact. -//! - A per-group sketch (KMV, HLL, KLL, DDSketch, Count-Min, Count Sketch, -//! UnivMon, at any size) read only by estimates whose merged readouts -//! keep a guarantee (not a heap's top-k or UnivMon's entropy). The coarse build becomes a rollup (a regrouping -//! `SummaryMerge`) of its finest finer build's states by the coarse -//! keys. The groups of one build partition its rows, so the rolled-up -//! state keeps the family's merged guarantee: the direct coarse build's -//! for KMV, HLL, DDSketch and the linear sketches, the same rank bound -//! for KLL. -//! -//! A sketch build over fewer rows than a finer one, whose extra conjuncts -//! read only the finer keys (`… WHERE city IN ('NYC', 'SF')` from a build -//! by `city`), is answered by first keeping the finer states that pass -//! those conjuncts, a compensating `Filter` over the states -//! ([`filtered_finer`], [`match_finer`](crate::pass2::finer_aggregate)), then -//! merging them. A SQL aggregate's finer *result* is read the same way by -//! the memo rule `memo::rules::finer_aggregate`. -//! -//! [`rollup_pass2_candidates`] returns one Pass 2 candidate per non-empty -//! subset of the rollup *choices* ([`rollup_choices`]): each exact rollup -//! alone, and every sketch rollup of one finest build together, since they -//! share its size. A chosen sketch choice sizes the finest build for every -//! reader, its own and each coarse one's (the strictest consumer, as the -//! shared-producer rule sizes a shared build), and composes every -//! guarantee above it again. So 15 groupings of one window are one choice, -//! not 2^14 subsets. -//! -//! # Exact top-k reuse rule -//! -//! Exact top-k computations of one input and partition with different `k` -//! read one ranking: the rows of the largest top-k, `Limit(k_max)` over -//! `Sort`, which hold every smaller top-k ([`topk_reuse_candidates`], -//! `memo::rules::topk`). Only exact computations over exact input are -//! reused; a sketch's heap is not, as its membership has no guarantee yet. - -use std::collections::HashMap; -use std::collections::HashSet; -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::scalar::resolve_column_ref; -use asap_types::ir::schema::{ - ExactKind, FamilyCapability, FieldDataType, GroupingStrategy, MergeInputRule, MergedError, - Schema, SketchKind, SummaryInputExpr, SummaryUpdate, WeightDomain, PROMQL_SERIES_IDENTITY, -}; -use asap_types::ir::ScalarExpr; -use asap_types::ir::SortKey; -use asap_types::ir::{ - ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, Predicate, QueryRoot, - TimeRangeKind, -}; - -use asap_accuracy::estimators::size_for_all_readers; -use asap_accuracy::propagation::recompose_guarantees; - -use asap_types::ir::SchemaDerivationError; - -/// Why a sharing rewrite builds no candidate. The rules report it as text -/// ([`Pass2RuleError`](crate::pass2::memo::state_groups::Pass2RuleError)) or -/// drop the candidate. -#[derive(Debug, thiserror::Error)] -pub enum CseError { - #[error(transparent)] - Structure(#[from] SchemaDerivationError), - #[error("unsupported substitution in logical ASAP-aware optimization: {0}")] - Unsupported(&'static str), - /// A rule found more than `max` candidates of one kind, whose 2^n subsets - /// it would enumerate: the workload is too large to plan this way, not - /// bad input. - #[error("found {found} {counted}, more than {max}")] - TooManyCandidates { - /// What was counted, e.g. "rollup choices". - counted: &'static str, - /// How many were found, more than `max`. - found: usize, - /// The most the rule enumerates. - max: usize, - }, -} - -/// `roots` with the time-series scans of one metric made identical when -/// they project different labels, so that interning finds them identical: the PromQL frontend projects only the labels each query uses -/// (`ts, value, job, …` for `quantile by (job) (0.5, data)`), and -/// structural CSE shares only nodes with equal schemas. Every scan of a -/// metric without predicates projects the union of the labels its scans -/// project, in the order the walk meets them (before the series identity, -/// with the unique keys moved along). -/// The nodes that pass the labels on (`passes_labels_on`) keep the wider -/// rows above a widened scan; any other consumer reads them through a -/// `Project` that keeps the columns it read, by name. `None` when every metric's scans already -/// agree, when two of them give one label different types or one is closed -/// and the other not, when a node would change arity, or when a consumer -/// would have to narrow a summary state or a grouped or filtered tumbling-window chain -/// would read a `Project`. -pub fn with_union_scan_labels(roots: &[QueryRoot]) -> Result>, CseError> { - let mut nodes = Vec::new(); - let mut seen = HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - // Each metric's union schema, `None` once two of its scans conflict. - let mut unions: Vec<(&Source, Option)> = Vec::new(); - for node in &nodes { - let Some((source, schema)) = unfiltered_series_scan(node) else { - continue; - }; - let Some(i) = unions.iter().position(|(s, _)| *s == source) else { - unions.push((source, Some(schema.clone()))); - continue; - }; - let Some(union) = unions[i].1.as_mut() else { - continue; - }; - // A scan known to hold every label and one that is not cannot share. - if union.closed != schema.closed { - unions[i].1 = None; - continue; - } - for field in &schema.fields { - match union.fields.iter().find(|f| f.name == field.name) { - Some(known) if known == field => {} - Some(_) => { - unions[i].1 = None; - break; - } - None => { - let at = union - .fields - .iter() - .position(|f| f.name == PROMQL_SERIES_IDENTITY) - .unwrap_or(union.fields.len()); - union.fields.insert(at, field.clone()); - // Unique keys (the series identity and ts) name columns - // by position. - for column in union.unique_keys.iter_mut().flatten() { - if *column >= at { - *column += 1; - } - } - } - } - } - } - let union_of = |node: &OperatorNode| { - let (source, schema) = unfiltered_series_scan(node)?; - let (_, union) = unions.iter().find(|(s, _)| *s == source)?; - union.as_ref().filter(|union| *union != schema) - }; - if !nodes.iter().any(|node| union_of(node).is_some()) { - return Ok(None); - } - let mut widened: HashMap<*const OperatorNode, Rc> = HashMap::new(); - for node in &nodes { - let out = match (union_of(node), node.non_asap()) { - (Some(union), Some(NonASAPOp::Scan { source, .. })) => { - let mut schema = union.clone(); - schema.time_index = schema.fields.iter().position(|f| f.name == "ts"); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: source.clone(), - predicates: Vec::new(), - schema, - }))? - } - _ if node - .children() - .iter() - .all(|c| Rc::ptr_eq(c, &widened[&Rc::as_ptr(c)])) => - { - Rc::clone(node) - } - // A node whose renamed outputs would change arity is not widened. - _ if passes_labels_on(node) => { - match node.with_new_children(|c| Rc::clone(&widened[&Rc::as_ptr(c)])) { - Ok(rebuilt) => Rc::new(rebuilt), - Err(_) => return Ok(None), - } - } - // A grouped or filtered tumbling-window chain over a `Project` would hide - // its window from plan selection. - _ if matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) => { - return Ok(None); - } - _ => { - let mut children = HashMap::new(); - for child in node.children() { - let wide = &widened[&Rc::as_ptr(child)]; - let Some(narrow) = narrowed(wide, &child.schema)? else { - return Ok(None); - }; - children.insert(Rc::as_ptr(child), narrow); - } - Rc::new(node.with_new_children(|c| Rc::clone(&children[&Rc::as_ptr(c)]))?) - } - }; - widened.insert(Rc::as_ptr(node), out); - } - let mut out = Vec::with_capacity(roots.len()); - for query_root in roots { - let mut mapped = HashMap::new(); - for node in query_root.operator_refs() { - let Some(narrow) = narrowed(&widened[&Rc::as_ptr(node)], &node.schema)? else { - return Ok(None); - }; - mapped.insert(Rc::as_ptr(node), narrow); - } - out.push(match query_root { - QueryRoot::Operator(node) => QueryRoot::Operator(Rc::clone(&mapped[&Rc::as_ptr(node)])), - QueryRoot::Scalar(expr) => QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| Rc::clone(&mapped[&Rc::as_ptr(node)])), - ), - }); - } - Ok(Some(recompose_guarantees(&out)?)) -} - -/// The source and schema of a time-series `Scan` without predicates. -fn unfiltered_series_scan(node: &OperatorNode) -> Option<(&Source, &Schema)> { - match node.non_asap() { - Some(NonASAPOp::Scan { - source: source @ Source::TimeSeries { .. }, - predicates, - schema, - }) if predicates.is_empty() => Some((source, schema)), - _ => None, - } -} - -/// Whether `node`, over a child with more labels, keeps them in its own -/// output: a time range or shift of the rows, and a per-series build or -/// tumbling-window chain without a filter (keyed by the series identity, which -/// determines every label) with its merges, estimates and finalizations. -/// The executor reads a per-series build's raw rows from its `TimeRange` -/// directly, and plan selection reads a tumbling-window chain's window from the rows -/// below it, so nothing may sit between them. -fn passes_labels_on(node: &OperatorNode) -> bool { - matches!( - node.non_asap(), - Some(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - ) || matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - reduction: Reduction::PerEntity, - filter: None, - .. - } | ASAPOp::SummaryTumblingWindowChain { - reduction: Reduction::PerEntity, - filter: None, - .. - } | ASAPOp::SummaryMerge { .. } - | ASAPOp::SummaryEstimate { .. } - | ASAPOp::FinalizeExactAccumulator { .. } - ) - ) -} - -/// `wide`, which holds every column of `schema` and perhaps more, with -/// only `schema`'s columns, by name: `wide` itself when it has exactly -/// them; `None` when `wide` is a summary state, which has no columns to -/// project. -fn narrowed( - wide: &Rc, - schema: &Schema, -) -> Result>, CseError> { - use asap_types::ir::ProjectItem; - if wide.schema.fields == schema.fields { - return Ok(Some(Rc::clone(wide))); - } - if wide.asap().is_some_and(|op| op.produced_state().is_some()) { - return Ok(None); - } - let cols = schema - .fields - .iter() - .map(|field| { - let column = wide - .schema - .fields - .iter() - .position(|f| f.name == field.name && f.table == field.table) - .expect("a widened node keeps every column it had"); - ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - } - }) - .collect(); - Ok(Some(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: Rc::clone(wide), - }, - ))?)) -} - -fn collect_children_first( - node: &Rc, - seen: &mut std::collections::HashSet<*const OperatorNode>, - out: &mut Vec>, -) { - if !seen.insert(Rc::as_ptr(node)) { - return; - } - for child in node.children() { - collect_children_first(child, seen, out); - } - out.push(Rc::clone(node)); -} - -/// Whether builds of families `a` and `b` can be one build: one sketch -/// algorithm and grouping at any sizes, or one exact accumulator. -fn same_build_family(a: &FieldDataType, b: &FieldDataType) -> bool { - match (a, b) { - (FieldDataType::Sketch(a, grouping), FieldDataType::Sketch(b, grouping_b)) => { - a.algorithm() == b.algorithm() && grouping == grouping_b - } - _ => a == b, - } -} - -/// One build serving every member: the first member with the members' -/// shared size ([`size_for_all_readers`]). -fn sized_for_all(members: &[&Rc]) -> Result>, CseError> { - // An exact accumulator has one size: the first member serves them all. - if matches!( - members[0].asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(..), - .. - }) - ) { - return Ok(Some(Rc::clone(members[0]))); - } - let kinds: Vec<&SketchKind> = members - .iter() - .filter_map(|member| match member.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind), - _ => None, - }) - .collect(); - let params: Vec<_> = kinds.iter().map(|kind| kind.params()).collect(); - let Some(params) = size_for_all_readers(¶ms) else { - return Ok(None); - }; - let first = members[0]; - let Some(ASAPOp::SummaryAgg { - child, - family: FieldDataType::Sketch(kind, grouping_strategy), - input, - reduction, - grouping, - filter, - }) = first.asap() - else { - return Ok(None); - }; - let operator = Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: FieldDataType::Sketch( - SketchKind::new(*kind.algorithm(), params), - grouping_strategy.clone(), - ), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }); - let mut build = OperatorNode::new(operator)?; - // Keep the member's explicit output names. - for (field, retained) in build.schema.fields.iter_mut().zip(&first.schema.fields) { - field.name = retained.name.clone(); - field.table = retained.table.clone(); - } - Ok(Some(Rc::new(build))) -} - -/// One summary build the window-composition rule can build as tumbling -/// tumbling windows: a `SummaryAgg` over a range `TimeRange(W)` whose family merges -/// (the family-capability table: exact accumulators included, and families -/// whose merged readouts are only approximate), read only by queries that -/// repeat at a fixed interval. Its tumbling windows are `tumbling_window_count = W / w` back-to-back -/// windows of `w = gcd(W, E)` over the queries' intervals `E` (and, for a -/// query with a range evaluation, its step), at least two. -#[derive(Debug, Clone)] -pub struct TumblingWindowCandidate { - /// The `SummaryAgg` over `TimeRange(W)` built as tumbling windows. - pub build: Rc, - /// The width `w` of one tumbling window. - pub tumbling_window_width_ms: u64, - /// The number of tumbling windows, `W / w`, at least two. - pub tumbling_window_count: u32, -} - -impl TumblingWindowCandidate { - /// The window `W` its tumbling windows cover, `tumbling_window_width_ms · tumbling_window_count`. - pub fn window_ms(&self) -> u64 { - self.tumbling_window_width_ms * u64::from(self.tumbling_window_count) - } -} - -/// A Pass 2 candidate of the window-composition rule: the candidate's roots -/// with the build of each of `tumbling_window_candidates` built as its tumbling windows, -/// merged before every read, and every guarantee composed again; the error -/// when that fails, which leaves the other Pass 2 candidates unaffected. -#[derive(Debug)] -pub struct WindowCompositionPass2Candidate { - /// The builds built as tumbling windows, in [`tumbling_window_candidates`] order. - /// Only tests read it. - #[cfg_attr(not(any(test, feature = "test-support")), allow(dead_code))] - pub tumbling_window_candidates: Vec, - /// The chains each shared by several of `tumbling_window_candidates` - /// ([`nested_window_groups`] or [`gcd_window_groups`]); empty when each - /// build has its own chain. - pub shared_chains: Vec, - /// The candidate's roots with those builds in tumbling windows. - pub roots: Result, CseError>, -} - -/// One tumbling-window chain several tumbling-window candidates read (window-composition rule): -/// tumbling windows of `tumbling_window_width_ms`, a width that divides every member's own tumbling window -/// width, and `tumbling_window_count` of them, covering the longest member's window. -/// Each member merges the newest `W / tumbling_window_width_ms` tumbling windows of its window `W`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SharedTumblingWindowChain { - /// Positions in the Pass 2 candidate's `tumbling_window_candidates`, two or more. - pub members: Vec, - /// The width of one tumbling window of the shared chain. - pub tumbling_window_width_ms: u64, - /// The number of tumbling windows of the shared chain. - pub tumbling_window_count: u32, -} - -/// The most tumbling-window candidates one candidate may have: every non-empty subset -/// is a Pass 2 candidate, and with shared grids up to three, about three -/// million at 20. -pub(crate) const MAX_TUMBLING_WINDOW_CANDIDATES: usize = 20; - -/// The tumbling-window candidates of the candidate with query roots `roots`, children -/// first. `evaluation_grid_ms[q]` is the grid query `q`'s evaluations fall -/// on: how often it repeats, or with a range evaluation the greatest common -/// divisor of that and its step, and of its phase if declared -/// (`QueryWorkloadEntry::evaluation_grid_ms`); `None` when it does not -/// repeat at a fixed interval. Each tumbling-window width divides every reader's grid, -/// so with tumbling windows anchored at the epoch every evaluation, every range step -/// included, ends on a tumbling-window boundary. -pub fn tumbling_window_candidates( - roots: &[QueryRoot], - evaluation_grid_ms: &[Option], -) -> Vec { - let mut nodes = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let reach: Vec> = roots - .iter() - .map(|root| { - root.operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .map(|node| Rc::as_ptr(&node)) - .collect() - }) - .collect(); - nodes - .into_iter() - .filter_map(|build| { - let window_ms = splittable_window_ms(&build)?; - let readers: Vec = (0..roots.len()) - .filter(|&q| reach[q].contains(&Rc::as_ptr(&build))) - .collect(); - let mut width = window_ms; - for &q in &readers { - width = asap_types::workload::greatest_common_divisor( - width, - evaluation_grid_ms.get(q).copied().flatten()?, - ); - } - let tumbling_window_count = u32::try_from(window_ms / width).ok()?; - (!readers.is_empty() && tumbling_window_count >= 1).then_some(TumblingWindowCandidate { - build, - tumbling_window_width_ms: width, - tumbling_window_count, - }) - }) - .collect() -} - -/// The window `W` of a build the rule can split: a `SummaryAgg` whose -/// family merges, over a range `TimeRange(W)` over an instant vector or a -/// SQL table's rows, read at the query's time or a fixed offset from it: no `@` anchor on any of -/// the time shifts stacked below the range. -fn splittable_window_ms(build: &OperatorNode) -> Option { - let Some(ASAPOp::SummaryAgg { child, family, .. }) = build.asap() else { - return None; - }; - FamilyCapability::of(family).merge_inputs?; - let Some(NonASAPOp::TimeRange { - range, - kind: TimeRangeKind::Range, - child: input, - }) = child.non_asap() - else { - return None; - }; - // An instant vector (PromQL `m[w]`) or a SQL table's rows (a trailing - // window); not a range vector, whose rows are already windows. - if !matches!( - input.result_kind, - OperatorResultKind::InstantVector | OperatorResultKind::Relation - ) { - return None; - } - // Refuse an `@` anchor on any of the shifts stacked below the range. - let mut below = input; - while let Some(NonASAPOp::TimeShift { shift, child }) = below.non_asap() { - if shift.at.is_some() { - return None; - } - below = child; - } - u64::try_from(range.as_millis()).ok().filter(|&ms| ms > 0) -} - -/// Every Pass 2 candidate of the window-composition rule for the candidate -/// with query roots `roots` and tumbling-window candidates `candidates` -/// ([`tumbling_window_candidates`]): one per non-empty subset of them, in -/// binary-counter order (the first candidate the lowest bit). Each build in -/// the subset becomes a `SummaryMerge` over one [`ASAPOp::SummaryTumblingWindowChain`] of -/// its tumbling windows; every other build is untouched. A subset whose builds can -/// share chains is followed by one Pass 2 candidate per shared grid, in -/// which each group reads one shared chain, each member merging its newest -/// tumbling windows: the finest member's tumbling windows (`nested_window_groups`), then the -/// gcd of the members' widths when that groups them differently -/// (`gcd_window_groups`); and by one error candidate per group whose -/// sizing fails. Each grid groups every build of a subset at once: a -/// shared gcd grid over only some of the builds that could share it is not -/// listed. An error only for more than -/// `MAX_TUMBLING_WINDOW_CANDIDATES` tumbling-window candidates; a Pass 2 candidate whose -/// composition fails carries its own error. -pub fn window_composition_pass2_candidates( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], -) -> Result, CseError> { - if candidates.len() > MAX_TUMBLING_WINDOW_CANDIDATES { - return Err(CseError::TooManyCandidates { - counted: "summary builds that could be built in tumbling windows", - found: candidates.len(), - max: MAX_TUMBLING_WINDOW_CANDIDATES, - }); - } - let rollable = rollable_tumbling_window_candidates(roots, candidates); - let mut pass2_candidates = Vec::new(); - for mask in 1u64..(1u64 << candidates.len()) { - let chosen: Vec = (0..candidates.len()) - .filter(|&i| mask >> i & 1 == 1) - .map(|i| candidates[i].clone()) - .collect(); - pass2_candidates.push(WindowCompositionPass2Candidate { - roots: in_tumbling_windows(roots, &chosen, &[]), - tumbling_window_candidates: chosen.clone(), - shared_chains: Vec::new(), - }); - let chosen_rollable: Vec = (0..candidates.len()) - .filter(|&i| mask >> i & 1 == 1) - .map(|i| rollable[i]) - .collect(); - push_every_shared_chain(roots, &chosen, &chosen_rollable, &mut pass2_candidates); - } - Ok(pass2_candidates) -} - -/// The Pass 2 candidates of [`window_composition_pass2_candidates`] in -/// which every one of `candidates` is in tumbling windows and reads a shared chain: -/// that function's candidates of the whole set with shared chains, in its -/// order, without building the other subsets'. -pub fn window_composition_shared_chain_candidates( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], -) -> Vec { - let rollable = rollable_tumbling_window_candidates(roots, candidates); - let mut pass2_candidates = Vec::new(); - push_every_shared_chain(roots, candidates, &rollable, &mut pass2_candidates); - pass2_candidates -} - -/// Push the shared-chain Pass 2 candidates of the subset `chosen` (each -/// rollable or not, `chosen_rollable`) of [`window_composition_pass2_candidates`]. -fn push_every_shared_chain( - roots: &[QueryRoot], - chosen: &[TumblingWindowCandidate], - chosen_rollable: &[bool], - pass2_candidates: &mut Vec, -) { - // Shared chains on the members' own finest tumbling windows, then, when it - // groups them differently, on the gcd of their widths. - let on_member_tumbling_windows = nested_window_groups(chosen); - let on_gcd_tumbling_windows = gcd_window_groups(chosen); - push_shared_chains(roots, chosen, &on_member_tumbling_windows, pass2_candidates); - if on_gcd_tumbling_windows != on_member_tumbling_windows { - push_shared_chains(roots, chosen, &on_gcd_tumbling_windows, pass2_candidates); - } - // Coarser groupings read a finer grouping's chain, rolled up. - push_shared_chains( - roots, - chosen, - &rollup_window_groups(chosen, chosen_rollable), - pass2_candidates, - ); -} - -/// Push the Pass 2 candidate in which each of `groups` reads its one shared -/// chain: the groups whose members have a shared size, and the error of -/// each group whose sizing fails. Nothing when no group has a shared size. -fn push_shared_chains( - roots: &[QueryRoot], - chosen: &[TumblingWindowCandidate], - groups: &[SharedTumblingWindowChain], - pass2_candidates: &mut Vec, -) { - // Each group with a size that serves every member, and that build. - let mut shared = Vec::new(); - for group in groups { - match shared_chain_build(chosen, &group.members) { - Ok(Some(build)) => shared.push((group.clone(), build)), - Ok(None) => {} - Err(error) => pass2_candidates.push(WindowCompositionPass2Candidate { - roots: Err(error), - tumbling_window_candidates: chosen.to_vec(), - shared_chains: vec![group.clone()], - }), - } - } - if !shared.is_empty() { - pass2_candidates.push(WindowCompositionPass2Candidate { - roots: in_tumbling_windows(roots, chosen, &shared), - tumbling_window_candidates: chosen.to_vec(), - shared_chains: shared.into_iter().map(|(group, _)| group).collect(), - }); - } -} - -/// The tumbling-window candidates (positions in `candidates`) that can read one -/// shared chain at the gcd of their tumbling-window widths, each group with its -/// chain: every build over one input ([`same_build_over_one_input`]) in one -/// group, whatever its tumbling-window width, on tumbling windows of the gcd of the members' -/// widths over the longest window. Finer tumbling windows than any member builds mean -/// more states and more merges; Stage 3's pricing decides whether sharing -/// pays. Groups of members with the same tumbling windows and window are left to the -/// other rules. -fn gcd_window_groups(candidates: &[TumblingWindowCandidate]) -> Vec { - let mut groups: Vec> = Vec::new(); - for (i, candidate) in candidates.iter().enumerate() { - match groups - .iter_mut() - .find(|group| same_build_over_one_input(&candidates[group[0]], candidate)) - { - Some(group) => group.push(i), - None => groups.push(vec![i]), - } - } - groups - .into_iter() - .filter_map(|members| { - let tumbling_windows = |i: usize| { - ( - candidates[i].tumbling_window_width_ms, - candidates[i].tumbling_window_count, - ) - }; - if members - .iter() - .all(|&i| tumbling_windows(i) == tumbling_windows(members[0])) - { - return None; - } - let tumbling_window_width_ms = members.iter().fold(0, |width, &i| { - asap_types::workload::greatest_common_divisor( - width, - candidates[i].tumbling_window_width_ms, - ) - }); - let longest = longest(candidates, &members); - let tumbling_window_count = - u32::try_from(candidates[longest].window_ms() / tumbling_window_width_ms).ok()?; - Some(SharedTumblingWindowChain { - members, - tumbling_window_width_ms, - tumbling_window_count, - }) - }) - .collect() -} - -/// The tumbling-window candidates (positions in `candidates`) that can read one -/// shared chain, each group with its chain: builds of one family -/// ([`same_family`], at any size) that read the same input below their -/// ranges the same way (item and weight, grouping and filter), with the -/// same output column names (each reader reads the shared chain's columns -/// by position). Their windows end at the same time, so each is the newest -/// tumbling windows of the longest. The shared tumbling windows are the group's finest member's -/// own tumbling windows: a build joins a group only when that width divides its own -/// tumbling-window width (the gcd of the widths, never finer than a chain some member -/// builds anyway), and otherwise starts its own group. Members at different -/// repeat intervals read the same tumbling windows, each at its own cadence. One chain -/// over the longest window, sized for every reader -/// ([`size_for_all_readers`]), serves them all. Groups of members with the -/// same tumbling windows and window are left to the other rules. -fn nested_window_groups(candidates: &[TumblingWindowCandidate]) -> Vec { - // Finest tumbling windows first: each build joins the first group over its input - // whose (finest) tumbling-window width divides its own, so a build whose tumbling windows no - // group's divide starts its own group instead of breaking another. - let mut by_width: Vec = (0..candidates.len()).collect(); - by_width.sort_by_key(|&i| candidates[i].tumbling_window_width_ms); - let mut groups: Vec> = Vec::new(); - for i in by_width { - let candidate = &candidates[i]; - match groups.iter_mut().find(|group| { - let finest = &candidates[group[0]]; - candidate - .tumbling_window_width_ms - .is_multiple_of(finest.tumbling_window_width_ms) - && same_build_over_one_input(finest, candidate) - }) { - Some(group) => group.push(i), - None => groups.push(vec![i]), - } - } - let mut shared = groups - .into_iter() - .filter_map(|mut members| { - let tumbling_windows = |i: usize| { - ( - candidates[i].tumbling_window_width_ms, - candidates[i].tumbling_window_count, - ) - }; - if members - .iter() - .all(|&i| tumbling_windows(i) == tumbling_windows(members[0])) - { - return None; - } - let tumbling_window_width_ms = candidates[members[0]].tumbling_window_width_ms; - let longest = longest(candidates, &members); - let tumbling_window_count = - u32::try_from(candidates[longest].window_ms() / tumbling_window_width_ms).ok()?; - members.sort_unstable(); - Some(SharedTumblingWindowChain { - members, - tumbling_window_width_ms, - tumbling_window_count, - }) - }) - .collect::>(); - // In the order of their first members, as [`gcd_window_groups`] lists - // its groups, so equal groupings compare equal. - shared.sort_by_key(|group| group.members[0]); - shared -} - -/// Whether each of `candidates` may be read rolled up from a finer -/// grouping's chain: a sketch build over a SQL relation's rows (a PromQL -/// rollup over newest tumbling windows is not offered yet) whose every reader in -/// `roots` is an estimate whose merged readout keeps a guarantee, as for -/// the rollup rule. -fn rollable_tumbling_window_candidates( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], -) -> Vec { - let mut nodes = Vec::new(); - let mut seen = HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let mut readers: HashMap<*const OperatorNode, Vec<&Rc>> = HashMap::new(); - for node in &nodes { - for child in node.children() { - readers.entry(Rc::as_ptr(child)).or_default().push(node); - } - } - candidates - .iter() - .map(|candidate| match candidate.build.asap() { - Some(ASAPOp::SummaryAgg { - child: range, - family: family @ FieldDataType::Sketch(..), - .. - }) => { - range.children()[0].result_kind == OperatorResultKind::Relation - && readers - .get(&Rc::as_ptr(&candidate.build)) - .is_some_and(|readers| read_with_merged_guarantee(family, readers)) - } - _ => false, - }) - .collect() -} - -/// The tumbling-window candidates (positions in `candidates`) that can read one -/// chain of a finer grouping's tumbling windows, each group with its chain: a host, -/// and every candidate whose group keys are the host's or fewer -/// ([`rolls_up_into`]), whose window is at most the host's and whose tumbling window -/// width the host's divides. Each member merges the newest tumbling windows of its -/// window, and a coarser member then rolls the host's groups up into its -/// own (a regrouping `SummaryMerge`): `p90 per city over 20 minutes` in -/// 10 s tumbling windows serves `p90 over 5 minutes` as the newest 30 tumbling windows merged -/// across cities. A coarser member is one `rollable[i]` allows -/// ([`rollable_tumbling_window_candidates`]). Hosts are tried by most group keys, -/// then longest window, and each candidate joins one group. Only groups -/// with a coarser member are listed; the others are -/// [`nested_window_groups`]'s. -fn rollup_window_groups( - candidates: &[TumblingWindowCandidate], - rollable: &[bool], -) -> Vec { - let mut hosts: Vec = (0..candidates.len()).collect(); - hosts.sort_by_key(|&i| { - ( - std::cmp::Reverse(group_key_count(&candidates[i].build)), - std::cmp::Reverse(candidates[i].window_ms()), - i, - ) - }); - let mut grouped = vec![false; candidates.len()]; - let mut groups = Vec::new(); - for host in hosts { - if grouped[host] { - continue; - } - let coarser = |i: usize| { - group_key_count(&candidates[i].build) < group_key_count(&candidates[host].build) - }; - let reads_host = |i: usize| { - let (member, chain) = (&candidates[i], &candidates[host]); - i == host - || (!grouped[i] - && member.window_ms() <= chain.window_ms() - && member - .tumbling_window_width_ms - .is_multiple_of(chain.tumbling_window_width_ms) - && rolls_up_into(member, chain) - && (!coarser(i) || rollable[i])) - }; - let members: Vec = (0..candidates.len()).filter(|&i| reads_host(i)).collect(); - if !members.iter().any(|&i| coarser(i)) { - continue; - } - for &i in &members { - grouped[i] = true; - } - groups.push(SharedTumblingWindowChain { - members, - tumbling_window_width_ms: candidates[host].tumbling_window_width_ms, - tumbling_window_count: candidates[host].tumbling_window_count, - }); - } - groups.sort_by_key(|group| group.members[0]); - groups -} - -/// The number of group keys of a build grouped by listed keys; 0 for any -/// other build. -fn group_key_count(build: &OperatorNode) -> usize { - match build.asap() { - Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(by), - .. - }) if !by.is_without() => by.len(), - _ => 0, - } -} - -/// Whether `member`'s build is `host`'s ([`same_build_over_one_input`]), -/// or the same but grouped by fewer of `host`'s keys, of a family whose -/// states merge across groups ([`FamilyCapability::merges_groups`]), so a -/// rollup of `host`'s states answers it. -fn rolls_up_into(member: &TumblingWindowCandidate, host: &TumblingWindowCandidate) -> bool { - if same_build_over_one_input(member, host) { - return true; - } - let ( - Some(ASAPOp::SummaryAgg { - child: range, - family, - input, - reduction: Reduction::Reduce(by), - grouping, - filter, - }), - Some(ASAPOp::SummaryAgg { - child: host_range, - family: host_family, - input: host_input, - reduction: Reduction::Reduce(host_by), - grouping: host_grouping, - filter: host_filter, - }), - ) = (member.build.asap(), host.build.asap()) - else { - return false; - }; - let (below, host_below) = (range.children()[0], host_range.children()[0]); - // Both ranges read the same rows below, so a key names one column. - !by.is_without() - && !host_by.is_without() - && by.len() < host_by.len() - && by.keys().iter().all(|key| host_by.keys().contains(key)) - && FamilyCapability::merges_groups(family) - && same_build_family(family, host_family) - && (Rc::ptr_eq(below, host_below) || below == host_below) - && input == host_input - && grouping == host_grouping - && filter == host_filter -} - -/// Whether two tumbling-window candidates build one family (at any size) over the -/// same input below their ranges, read the same way, with the same output -/// column names. -fn same_build_over_one_input(a: &TumblingWindowCandidate, b: &TumblingWindowCandidate) -> bool { - let ( - Some(ASAPOp::SummaryAgg { - child: range, - family, - input, - reduction, - grouping, - filter, - }), - Some(ASAPOp::SummaryAgg { - child: range_b, - family: family_b, - input: input_b, - reduction: reduction_b, - grouping: grouping_b, - filter: filter_b, - }), - ) = (a.build.asap(), b.build.asap()) - else { - return false; - }; - let (below, below_b) = (range.children()[0], range_b.children()[0]); - let names = |build: &OperatorNode| -> Vec<(String, Option)> { - build - .schema - .fields - .iter() - .map(|f| (f.name.clone(), f.table.clone())) - .collect() - }; - names(&a.build) == names(&b.build) - && same_build_family(family, family_b) - && (Rc::ptr_eq(below, below_b) || below == below_b) - && input == input_b - && reduction == reduction_b - && grouping == grouping_b - && filter == filter_b -} - -/// The member of `group` (positions in `candidates`) with the longest -/// window. -fn longest(candidates: &[TumblingWindowCandidate], group: &[usize]) -> usize { - *group - .iter() - .max_by_key(|&&i| candidates[i].window_ms()) - .expect("a group has members") -} - -/// The build one chain shared by `group` (positions in `candidates`) is -/// made of: the longest member's, sized for every member -/// ([`sized_for_all`]); `None` when their sizes have no shared size. -fn shared_chain_build( - candidates: &[TumblingWindowCandidate], - group: &[usize], -) -> Result>, CseError> { - // Of the longest windows, the finest grouping: a rollup group's host. - let host = *group - .iter() - .max_by_key(|&&i| { - ( - candidates[i].window_ms(), - group_key_count(&candidates[i].build), - ) - }) - .expect("a group has members"); - let members: Vec<&Rc> = std::iter::once(host) - .chain(group.iter().copied().filter(|&i| i != host)) - .map(|i| &candidates[i].build) - .collect(); - sized_for_all(&members) -} - -/// `roots` with the build of each of `candidates` built as its tumbling -/// tumbling windows, and every guarantee composed again. The members of each -/// [`SharedTumblingWindowChain`] (with its [`shared_chain_build`]) read that one -/// chain: each merges the newest tumbling windows covering its own window. -fn in_tumbling_windows( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], - shared_chains: &[(SharedTumblingWindowChain, Rc)], -) -> Result, CseError> { - let mut replaced: HashMap<*const OperatorNode, Rc> = HashMap::new(); - for (shared, build) in shared_chains { - let chain = tumbling_window_chain(&TumblingWindowCandidate { - build: Rc::clone(build), - tumbling_window_width_ms: shared.tumbling_window_width_ms, - tumbling_window_count: shared.tumbling_window_count, - })?; - for &i in &shared.members { - let newest = u32::try_from(candidates[i].window_ms() / shared.tumbling_window_width_ms) - .expect("a member's window is at most the chain's"); - let merge = Rc::new(OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&chain)], - newest_tumbling_windows: (newest < shared.tumbling_window_count).then_some(newest), - group_by: None, - }))?); - // A coarser member rolls the chain's groups up into its own. - let member = &candidates[i].build; - let read = if group_key_count(member) == group_key_count(build) { - merge - } else { - regrouped(member, &merge).ok_or(CseError::Unsupported( - "a coarser tumbling-window build whose keys the shared chain does not hold", - ))? - }; - replaced.insert(Rc::as_ptr(member), read); - } - } - for candidate in candidates { - if let std::collections::hash_map::Entry::Vacant(entry) = - replaced.entry(Rc::as_ptr(&candidate.build)) - { - entry.insert(Rc::new(OperatorNode::new(Operator::ASAP( - ASAPOp::SummaryMerge { - children: vec![tumbling_window_chain(candidate)?], - newest_tumbling_windows: None, - group_by: None, - }, - ))?)); - } - } - let mut rebuilt: HashMap<*const OperatorNode, Rc> = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(rebuild(node, &replaced, &mut rebuilt)?) - } - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), rebuild(node, &replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// `roots` with the `SummaryMerge` `merge` (over one [`ASAPOp::SummaryTumblingWindowChain`] -/// of `roots`) reading the newest tumbling windows of another chain over the same -/// input: `tumbling_window_count` tumbling windows of `tumbling_window_width_ms` of family `family` (a size -/// that serves every reader of the chain). The merge reads the newest -/// tumbling windows covering its own window (its newest tumbling windows, or its whole chain), -/// which `tumbling_window_width_ms` divides and the new chain covers; every guarantee -/// is composed again. -pub(crate) fn with_tumbling_window_chain( - roots: &[QueryRoot], - merge: &Rc, - family: FieldDataType, - tumbling_window_width_ms: u64, - tumbling_window_count: u32, -) -> Result, CseError> { - let Some(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: own_newest, - .. - }) = merge.asap() - else { - unreachable!("a merge over a tumbling-window chain"); - }; - let own = &children[0]; - let Some(ASAPOp::SummaryTumblingWindowChain { - child, - input, - reduction, - grouping, - filter, - tumbling_window_width_ms: own_width, - tumbling_window_count: own_count, - .. - }) = own.asap() - else { - unreachable!("a merge over a tumbling-window chain"); - }; - // The merge's window: its newest tumbling windows, or the whole chain. - let window_ms = own_width * u64::from(own_newest.unwrap_or(*own_count)); - let newest = u32::try_from(window_ms / tumbling_window_width_ms) - .expect("a reader's window is at most the chain's"); - let mut chain = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(child), - family, - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - tumbling_window_width_ms, - tumbling_window_count, - }))?; - for (field, kept) in chain.schema.fields.iter_mut().zip(&own.schema.fields) { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - let read = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::new(chain)], - newest_tumbling_windows: (newest < tumbling_window_count).then_some(newest), - group_by: None, - }))?; - let replaced = HashMap::from([(Rc::as_ptr(merge), Rc::new(read))]); - let mut rebuilt = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(rebuild(node, &replaced, &mut rebuilt)?) - } - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), rebuild(node, &replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// `roots` with the summary build `build` (a `SummaryAgg` of `roots`) -/// built at family `family` instead (the same algorithm at another size), -/// its output names kept; every guarantee is composed again. -pub(crate) fn with_resized_build( - roots: &[QueryRoot], - build: &Rc, - family: FieldDataType, -) -> Result, CseError> { - let Some(ASAPOp::SummaryAgg { - child, - input, - reduction, - grouping, - filter, - .. - }) = build.asap() - else { - unreachable!("a summary build"); - }; - let mut resized = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family, - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }))?; - for (field, kept) in resized.schema.fields.iter_mut().zip(&build.schema.fields) { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - let replaced = HashMap::from([(Rc::as_ptr(build), Rc::new(resized))]); - let mut rebuilt = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(rebuild(node, &replaced, &mut rebuilt)?) - } - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), rebuild(node, &replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// The [`ASAPOp::SummaryTumblingWindowChain`] of `candidate`'s tumbling windows, with the build's -/// output names. -fn tumbling_window_chain( - candidate: &TumblingWindowCandidate, -) -> Result, CseError> { - let Some(ASAPOp::SummaryAgg { - child: range, - family, - input, - reduction, - grouping, - filter, - }) = candidate.build.asap() - else { - unreachable!("a tumbling-window candidate's build is a SummaryAgg"); - }; - let mut chain = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(range.children()[0]), - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - tumbling_window_width_ms: candidate.tumbling_window_width_ms, - tumbling_window_count: candidate.tumbling_window_count, - }))?; - for (field, kept) in chain - .schema - .fields - .iter_mut() - .zip(&candidate.build.schema.fields) - { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - Ok(Rc::new(chain)) -} - -/// `node` with every replaced build below it swapped for its shared build; -/// the same `Rc` when nothing below changed. -fn rebuild( - node: &Rc, - replaced: &HashMap<*const OperatorNode, Rc>, - rebuilt: &mut HashMap<*const OperatorNode, Rc>, -) -> Result, CseError> { - if let Some(done) = rebuilt.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(done)); - } - let out = match replaced.get(&Rc::as_ptr(node)) { - Some(build) => Rc::clone(build), - None => { - let mut changed = false; - let mut children = HashMap::new(); - for child in node.children() { - let new = rebuild(child, replaced, rebuilt)?; - changed |= !Rc::ptr_eq(&new, child); - children.insert(Rc::as_ptr(child), new); - } - if changed { - Rc::new(node.with_new_children(|child| children[&Rc::as_ptr(child)].clone())?) - } else { - Rc::clone(node) - } - } - }; - rebuilt.insert(Rc::as_ptr(node), Rc::clone(&out)); - Ok(out) -} - -/// One coarse build the rollup rule can answer from a finer one: the -/// coarse `SummaryAgg`, the finer node it reads, and the node that replaces -/// the coarse value or state. -#[derive(Debug, Clone)] -pub struct RollupCandidate { - /// The coarse build (`SummaryAgg`). An exact accumulator's every - /// reader is a `FinalizeExactAccumulator`, which the rollup replaces; a - /// sketch's every reader is an estimate its family answers, and the - /// rollup replaces the build itself. - pub coarse: Rc, - /// For an exact accumulator, the finer build's - /// `FinalizeExactAccumulator`, whose values the rollup reads; for a - /// sketch, the finest build of the same algorithm over the same rows - /// (a `SummaryAgg`), whose states the rollup merges. Both are already - /// in the candidate. - pub finer: Rc, - /// For an exact accumulator, the finalized exact accumulator over - /// `finer`'s values, grouped by the coarse keys, with the coarse value's - /// column names; for a sketch, the regrouping `SummaryMerge` of - /// `finer`'s states by the coarse keys, at `finer`'s size (a Pass 2 - /// candidate resizes `finer` for every reader). - pub rollup: Rc, -} - -/// A Pass 2 candidate of the rollup rule: the candidate's roots with the -/// coarse build of each chosen rollup candidate answered from its finer -/// build, and every guarantee composed again; the error when that fails. -#[derive(Debug)] -pub struct RollupPass2Candidate { - /// The candidate's roots with those builds rolled up and every guarantee - /// composed again; `Err` when the rebuilt DAG does not derive or - /// compose, which drops only this candidate. - pub roots: Result, CseError>, -} - -/// The most rollup or top-k reuse candidates one candidate may have: every -/// non-empty subset is a Pass 2 candidate, about a million at 20. -pub(crate) const MAX_REUSE_CANDIDATES: usize = 20; - -/// The exact accumulator that merges finalized values of `family` across -/// disjoint groups, by the family-capability table: a family whose states -/// merge over disjoint (or overlapping) selections into an exact state. -/// `None` for a family without such a merge: counter states merge only -/// consecutive windows, and sketch states have no regrouping merge in the IR. -pub(crate) fn rollup_kind(family: &FieldDataType) -> Option { - let FieldDataType::ExactAggregate(kind, _) = family else { - return None; - }; - // An exact accumulator's merged state is exact; a counter state merges - // only consecutive windows. - if FamilyCapability::of(family).merge_inputs? == MergeInputRule::DisjointInTime { - return None; - } - match kind { - // Counts add up. - ExactKind::Sum | ExactKind::Count => Some(ExactKind::Sum), - ExactKind::Min => Some(ExactKind::Min), - ExactKind::Max => Some(ExactKind::Max), - ExactKind::Rate | ExactKind::IRate | ExactKind::Increase => None, - } -} - -/// The rollup candidates of the candidate with query roots `roots`, children -/// first: each coarse exact accumulator build read only through its -/// finalized value, with its coarsest finer build of the same family over -/// the same rows that the candidate finalizes; then each coarse sketch build -/// read only by estimates, with the finest build of its algorithm over the -/// same rows whose keys include its own (`sketch_rollup_candidates`). -#[cfg(any(test, feature = "test-support"))] -pub fn rollup_candidates(roots: &[QueryRoot]) -> Vec { - rollup_candidates_of(roots, None) -} - -/// The rollup candidate of coarse build `coarse` among -/// [`rollup_candidates`]`(roots)`, the first when several, computed for that -/// build alone (each coarse build's candidate depends only on `roots`, not -/// on the other coarse builds). -pub(crate) fn rollup_candidate_for( - roots: &[QueryRoot], - coarse: *const OperatorNode, -) -> Option { - rollup_candidates_of(roots, Some(coarse)).into_iter().next() -} - -/// [`rollup_candidates`], over the coarse builds that are `only` when set. -fn rollup_candidates_of( - roots: &[QueryRoot], - only: Option<*const OperatorNode>, -) -> Vec { - let mut nodes = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let mut readers: HashMap<*const OperatorNode, Vec<&Rc>> = HashMap::new(); - for node in &nodes { - for child in node.children() { - readers.entry(Rc::as_ptr(child)).or_default().push(node); - } - } - let finalized = |build: &Rc| -> Vec<&Rc> { - readers - .get(&Rc::as_ptr(build)) - .map_or_else(Vec::new, |readers| { - readers - .iter() - .filter(|r| matches!(r.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. }))) - .copied() - .collect() - }) - }; - let builds: Vec<&Rc> = nodes.iter().filter(|n| grouped_exact(n)).collect(); - let wanted = |coarse: &Rc| only.is_none_or(|only| Rc::as_ptr(coarse) == only); - let mut candidates = Vec::new(); - for &coarse in builds.iter().filter(|coarse| wanted(coarse)) { - let finals = finalized(coarse); - // Every reader of the coarse state reads its finalized value. - if finals.is_empty() || finals.len() != readers[&Rc::as_ptr(coarse)].len() { - continue; - } - let mut best: Option<(usize, RollupCandidate)> = None; - for &fine in &builds { - let Some(keys) = finer_keys(coarse, fine) else { - continue; - }; - if best.as_ref().is_some_and(|(k, _)| *k <= keys) { - continue; - } - let Some(&finer) = finalized(fine).first() else { - continue; - }; - if let Some(rollup) = rolled_up(coarse, finer, finals[0]) { - best = Some(( - keys, - RollupCandidate { - coarse: Rc::clone(coarse), - finer: Rc::clone(finer), - rollup, - }, - )); - } - } - candidates.extend(best.map(|(_, candidate)| candidate)); - } - candidates.extend(sketch_rollup_candidates(&nodes, &readers, only)); - candidates -} - -/// The sketch rollup candidates among `nodes` (children first), whose -/// readers are `readers`: each per-group sketch build grouped by listed keys -/// and read only by estimates its family answers, answered from the finest -/// build ([`finer_keys`] with the most keys, and no build finer than it) of -/// the same algorithm (at any size), update and filter over the same rows, -/// whose family's states merge into a state with a guarantee -/// ([`regroupable`]). Every coarse build of one finest build is one -/// choice in [`rollup_pass2_candidates`], so the choices stay few however -/// many groupings a workload asks for. -fn sketch_rollup_candidates( - nodes: &[Rc], - readers: &HashMap<*const OperatorNode, Vec<&Rc>>, - only: Option<*const OperatorNode>, -) -> Vec { - let builds: Vec<&Rc> = nodes.iter().filter(|n| regroupable(n)).collect(); - // The finer build with the most keys; among several (groupings that - // are not nested), the one whose sorted key names come first, so the - // choice does not depend on query order. - let key_names = |build: &OperatorNode| -> Vec { - let mut names: Vec = build - .schema - .fields - .iter() - .filter(|field| field.is_plain()) - .map(column_name) - .collect(); - names.sort(); - names - }; - let finest = |coarse: &OperatorNode| { - builds - .iter() - .filter_map(|&fine| Some((finer_keys(coarse, fine)?, fine))) - .min_by(|(keys, fine), (other_keys, other)| { - other_keys - .cmp(keys) - .then_with(|| key_names(fine).cmp(&key_names(other))) - }) - .map(|(_, fine)| fine) - }; - let mut candidates = Vec::new(); - for &coarse in builds - .iter() - .filter(|coarse| only.is_none_or(|only| Rc::as_ptr(coarse) == only)) - { - let Some(ASAPOp::SummaryAgg { family, .. }) = coarse.asap() else { - continue; - }; - if !readers - .get(&Rc::as_ptr(coarse)) - .is_some_and(|readers| read_with_merged_guarantee(family, readers)) - { - continue; - } - // A build finer than the one with the most keys would have more. - // Over more rows, the finer build's states are first filtered by - // the coarse build's extra conjuncts (a compensating filter). - let rollup = match finest(coarse) { - Some(fine) => regrouped(coarse, fine).map(|rollup| (fine, rollup)), - None => builds - .iter() - .filter_map(|&fine| Some((filtered_finer(coarse, fine)?, fine))) - // The same choice as `finest`'s, whatever the query order. - .min_by(|((keys, _), fine), ((other_keys, _), other)| { - other_keys - .cmp(keys) - .then_with(|| key_names(fine).cmp(&key_names(other))) - }) - .and_then(|((_, pred), fine)| { - let kept = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred, - child: Rc::clone(fine), - })) - .ok()?; - Some((fine, regrouped(coarse, &kept)?)) - }), - }; - if let Some((fine, rollup)) = rollup { - candidates.push(RollupCandidate { - coarse: Rc::clone(coarse), - finer: Rc::clone(fine), - rollup, - }); - } - } - candidates -} - -/// Whether every one of `readers`, the readers of a build of `family`, is -/// an estimate whose merged readout keeps a guarantee: not a heap's top-k -/// or a UnivMon's entropy. -fn read_with_merged_guarantee(family: &FieldDataType, readers: &[&Rc]) -> bool { - let capability = FamilyCapability::of(family); - readers.iter().all(|reader| match reader.asap() { - Some(ASAPOp::SummaryEstimate { query, .. }) => { - capability.answers(query) - && matches!( - capability.merged_error(query), - Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) - ) - } - _ => false, - }) -} - -/// Whether `node` is a per-group sketch build grouped by listed keys whose -/// states a rollup may merge across groups -/// ([`FamilyCapability::merges_groups`]). -fn regroupable(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: family @ FieldDataType::Sketch(..), - reduction: Reduction::Reduce(by), - .. - }) if !by.is_without() && FamilyCapability::merges_groups(family) - ) -} - -/// The regrouping `SummaryMerge` of `fine`'s states by `coarse`'s keys, in -/// `coarse`'s order and with its column names; `None` when a key is not in -/// `fine`'s state. -fn regrouped(coarse: &OperatorNode, fine: &Rc) -> Option> { - let Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(by), - .. - }) = coarse.asap() - else { - return None; - }; - let keys = (0..by.keys().len()) - .map(|key| { - let name = column_name(&coarse.schema.fields[key]); - fine.schema - .fields - .iter() - .position(|field| field.is_plain() && column_name(field) == name) - }) - .collect::>>()?; - regroup_merge(fine, keys, coarse).ok() -} - -/// `SummaryMerge { [state], group_by: keys }`, its columns named as -/// `named`'s (the coarse build it stands for). -fn regroup_merge( - state: &Rc, - keys: Vec, - named: &OperatorNode, -) -> Result, asap_types::ir::SchemaDerivationError> { - let merge = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(state)], - group_by: Some(keys), - newest_tumbling_windows: None, - }))?; - let mut schema = merge.schema.clone(); - for (field, coarse) in schema.fields.iter_mut().zip(&named.schema.fields) { - field.name = coarse.name.clone(); - field.table = coarse.table.clone(); - } - Ok(Rc::new(OperatorNode::with_schema(merge.operator, schema))) -} - -/// Whether `node` is an exact accumulator build grouped by listed keys whose -/// family merges across groups ([`rollup_kind`]). -fn grouped_exact(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family, - reduction: Reduction::Reduce(by), - grouping: GroupingStrategy::PerSubpopulationInstance, - .. - }) if !by.is_without() && rollup_kind(family).is_some() - ) -} - -/// The number of group keys of `fine` when it can answer `coarse`: the same -/// family (a sketch's algorithm at any size), update and filter over the -/// same rows, and group keys (by column) that strictly include the coarse -/// ones. -fn finer_keys(coarse: &OperatorNode, fine: &OperatorNode) -> Option { - let ( - Some(ASAPOp::SummaryAgg { - child, - family, - input, - reduction: Reduction::Reduce(by), - filter, - .. - }), - Some(ASAPOp::SummaryAgg { - child: fine_child, - family: fine_family, - input: fine_input, - reduction: Reduction::Reduce(fine_by), - filter: fine_filter, - .. - }), - ) = (coarse.asap(), fine.asap()) - else { - return None; - }; - let names = |node: &OperatorNode, keys: &[usize]| -> Option> { - keys.iter() - .map(|&k| node.schema.fields.get(k).map(column_name)) - .collect() - }; - let coarse_keys = names(child, by.keys())?; - let fine_keys = names(fine_child, fine_by.keys())?; - let same_filter = match (filter, fine_filter) { - (None, None) => true, - // A filter indexes its input's columns: equal only over equal rows. - (a, b) => a == b && (Rc::ptr_eq(child, fine_child) || child == fine_child), - }; - (same_build_family(family, fine_family) - && input == fine_input - && same_filter - && fine_keys.len() > coarse_keys.len() - && coarse_keys.iter().all(|key| fine_keys.contains(key)) - && same_rows(child, fine_child)) - .then_some(fine_keys.len()) -} - -/// The number of group keys of `fine` and the compensating filter over its -/// states when `fine` answers `coarse` from more rows: the same family, -/// update and no build filter, over rows that hold `coarse`'s, whose extra -/// conjuncts read only `fine`'s keys ([`match_finer`]), with group keys that -/// include the coarse ones. `None` over the same rows ([`finer_keys`]). -fn filtered_finer(coarse: &OperatorNode, fine: &OperatorNode) -> Option<(usize, Predicate)> { - let ( - Some(ASAPOp::SummaryAgg { - child, - family, - input, - reduction: Reduction::Reduce(by), - filter: None, - .. - }), - Some(ASAPOp::SummaryAgg { - child: fine_child, - family: fine_family, - input: fine_input, - reduction: Reduction::Reduce(fine_by), - filter: None, - .. - }), - ) = (coarse.asap(), fine.asap()) - else { - return None; - }; - if by.is_without() - || fine_by.is_without() - || !same_build_family(family, fine_family) - || input != fine_input - { - return None; - } - let matched = - crate::pass2::finer_aggregate::match_finer(child, by.keys(), fine_child, fine_by.keys())?; - // The filter reads the finer keys as the state's columns of those names. - let state_columns = fine_by - .keys() - .iter() - .map(|&key| { - // A state's key column keeps the input column's name, not - // always its table qualifier. - let name = &fine_child.schema.fields.get(key)?.name; - fine.schema - .fields - .iter() - .position(|field| field.is_plain() && field.name == *name) - }) - .collect::>>()?; - let pred = matched.compensating_filter?; - Some(( - fine_by.keys().len(), - Predicate(pred.0.map_columns(&mut |key| state_columns[key])), - )) -} - -type ColumnName = (Option, String); - -fn column_name(field: &asap_types::ir::schema::Field) -> ColumnName { - (field.table.clone(), field.name.clone()) -} - -/// Whether `a` and `b` output the same rows: the same or equal nodes, or -/// time-series scans of one metric without predicates (projecting perhaps -/// other labels) under the same time ranges and shifts. -fn same_rows(a: &Rc, b: &Rc) -> bool { - if Rc::ptr_eq(a, b) || a == b { - return true; - } - match (a.non_asap(), b.non_asap()) { - ( - Some(NonASAPOp::Scan { - source: source @ Source::TimeSeries { .. }, - predicates, - .. - }), - Some(NonASAPOp::Scan { - source: other, - predicates: other_predicates, - .. - }), - ) => source == other && predicates.is_empty() && other_predicates.is_empty(), - ( - Some(NonASAPOp::TimeRange { range, kind, child }), - Some(NonASAPOp::TimeRange { - range: other_range, - kind: other_kind, - child: other, - }), - ) => range == other_range && kind == other_kind && same_rows(child, other), - ( - Some(NonASAPOp::TimeShift { shift, child }), - Some(NonASAPOp::TimeShift { - shift: other_shift, - child: other, - }), - ) => shift == other_shift && same_rows(child, other), - ( - Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures, - filters, - having: None, - child, - .. - }), - Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: other_measures, - filters: other_filters, - having: None, - child: other, - .. - }), - ) => { - // These reductions preserve series identity and read the implicit - // sample value. Extra projected labels do not change their values. - // Explicit column indices and filtered measures need remapping and - // are deliberately excluded. - measures == other_measures - && matches!( - measures.as_slice(), - [AggIntent::Avg { col: None } - | AggIntent::Sum { col: None } - | AggIntent::Min { col: None } - | AggIntent::Max { col: None }] - ) - && filters.iter().all(Option::is_none) - && other_filters.iter().all(Option::is_none) - && same_rows(child, other) - } - _ => false, - } -} - -/// The finalized value of `coarse` (`coarse_value`) as an exact -/// accumulator over `finer` (the finer build's finalized values), grouped by -/// the coarse keys, finalized, with `coarse_value`'s column names; `None` -/// when it cannot be built, a non-count finer value is not Float64 (the -/// supported accumulator update type), or it differs from -/// `coarse_value` in type. -fn rolled_up( - coarse: &OperatorNode, - finer: &Rc, - coarse_value: &OperatorNode, -) -> Option> { - let Some(ASAPOp::SummaryAgg { - child, - family, - reduction, - .. - }) = coarse.asap() - else { - return None; - }; - let Some(ASAPOp::FinalizeExactAccumulator { child: fine }) = finer.asap() else { - return None; - }; - let Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(fine_by), - .. - }) = fine.asap() - else { - return None; - }; - // A grouped accumulator outputs its keys, then its value. - let value = fine_by.len(); - // Merge count states directly: converting finalized Int64 counts through - // a Float64 sum would lose integer precision above 2^53. - if matches!(family, FieldDataType::ExactAggregate(ExactKind::Count, _)) { - let merged = regrouped(coarse, fine)?; - let finalized = - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: merged, - })) - .ok()?; - if finalized.schema.fields.len() != coarse_value.schema.fields.len() - || finalized - .schema - .fields - .iter() - .zip(&coarse_value.schema.fields) - .any(|(a, b)| a.dtype != b.dtype || a.nullable != b.nullable) - || finalized.schema.unique_keys != coarse_value.schema.unique_keys - { - return None; - } - // Finalization preserves the count's type; only the reader's SQL alias - // differs from the canonical state readout name. - return Some(Rc::new(OperatorNode::with_schema( - finalized.operator.clone(), - coarse_value.schema.clone(), - ))); - } - if finer.schema.fields.get(value).map(|f| &f.dtype) - != Some(&FieldDataType::Plain( - asap_types::ir::schema::DataType::Float64, - )) - { - return None; - } - let regrouped = reduction.partitions_over(&child.schema, &finer.schema)?; - if regrouped - .group_keys() - .is_some_and(|keys| keys.iter().any(|&k| k >= value)) - { - return None; - } - let kind = rollup_kind(family)?; - // An exact accumulator over the finer value column: what Pass 1 builds - // for a Sum, Min or Max of that column (no item, signed weights). - let weight = finer.schema.fields[value].column_ref(); - resolve_column_ref(&weight, &finer.schema).ok()?; - let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(finer), - input: SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(weight), - weight_domain: WeightDomain::UnknownOrSigned, - }, - family: FieldDataType::exact(kind), - reduction: regrouped, - grouping: GroupingStrategy::default(), - filter: None, - })) - .ok()?; - let mut finalized = - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: state, - })) - .ok()?; - let types = |node: &OperatorNode| -> Vec { - node.schema.fields.iter().map(|f| f.dtype.clone()).collect() - }; - if types(&finalized) != types(coarse_value) { - return None; - } - // A count summed is named after the sum: rename to the coarse columns. - if finalized.schema.fields != coarse_value.schema.fields { - let cols = coarse_value - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| asap_types::ir::ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(); - finalized = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: finalized, - })) - .ok()?; - } - Some(finalized) -} - -/// The independent choices among rollup candidates `candidates` -/// ([`rollup_candidates`]), in order: each exact accumulator's alone, and -/// every sketch candidate of one finest build together, since they share -/// that build's size. -pub(crate) fn rollup_choices(candidates: &[RollupCandidate]) -> Vec> { - let mut choices: Vec> = Vec::new(); - for candidate in candidates { - let sketch = is_sketch_build(&candidate.finer); - match choices.iter_mut().find(|choice| { - sketch - && is_sketch_build(&choice[0].finer) - && Rc::ptr_eq(&choice[0].finer, &candidate.finer) - }) { - Some(choice) => choice.push(candidate.clone()), - None => choices.push(vec![candidate.clone()]), - } - } - choices -} - -fn is_sketch_build(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) -} - -/// Every Pass 2 candidate of the rollup rule for the candidate with query -/// roots `roots` and rollup candidates `candidates` ([`rollup_candidates`]): -/// one per non-empty subset of their choices (`rollup_choices`), in -/// binary-counter order (the first the lowest bit). A chosen sketch choice -/// resizes its finest build for every reader, its own and each coarse -/// build's (`sized_for_all`: the strictest consumer's size), and each -/// coarse build becomes the regrouping merge of the resized build's -/// states. An error only for more than `MAX_REUSE_CANDIDATES` choices; a -/// Pass 2 candidate whose composition fails carries its own error. -pub fn rollup_pass2_candidates( - roots: &[QueryRoot], - candidates: &[RollupCandidate], -) -> Result, CseError> { - Ok(subsets(&rollup_choices(candidates), "rollup")? - .into_iter() - .map(|choices| { - let values: HashMap<*const OperatorNode, &RollupCandidate> = choices - .iter() - .flatten() - .filter(|candidate| !is_sketch_build(&candidate.finer)) - .map(|candidate| (Rc::as_ptr(&candidate.coarse), candidate)) - .collect(); - // Every finalized value of a chosen coarse exact build is - // replaced. - let mut replaced: HashMap<*const OperatorNode, Rc> = roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|node| match node.asap() { - Some(ASAPOp::FinalizeExactAccumulator { child }) => { - let candidate = values.get(&Rc::as_ptr(child))?; - Some((Rc::as_ptr(&node), Rc::clone(&candidate.rollup))) - } - _ => None, - }) - .collect(); - let roots = sketch_rollups(&choices).and_then(|sketches| { - replaced.extend(sketches); - replace_all(roots, &replaced) - }); - RollupPass2Candidate { roots } - }) - .collect()) -} - -/// The replacements of the sketch rollup choices among `choices` -/// ([`rollup_choices`]): each finest build by the build sized for it and -/// every coarse build it serves, and each coarse build by the regrouping -/// merge of that build's states. An error when a finest build and its -/// coarse builds have no shared size, or a node does not derive. -fn sketch_rollups( - choices: &[Vec], -) -> Result)>, CseError> { - let unsupported = || { - CseError::Unsupported("a sketch rollup whose finest and coarse builds have no shared size") - }; - // A finest build can also be the coarse build of another choice, read - // through a compensating filter. Each build is replaced once: as a - // finest build, sized for its readers. - let finest: HashSet<*const OperatorNode> = choices - .iter() - .filter(|choice| is_sketch_build(&choice[0].finer)) - .map(|choice| Rc::as_ptr(&choice[0].finer)) - .collect(); - let mut replaced = Vec::new(); - for choice in choices { - let fine = &choice[0].finer; - if !is_sketch_build(fine) { - continue; - } - let mut members = vec![fine]; - members.extend(choice.iter().map(|candidate| &candidate.coarse)); - let sized = sized_for_all(&members)?.ok_or_else(unsupported)?; - replaced.push((Rc::as_ptr(fine), Rc::clone(&sized))); - for candidate in choice { - if finest.contains(&Rc::as_ptr(&candidate.coarse)) { - continue; - } - let Some(ASAPOp::SummaryMerge { - group_by: Some(keys), - .. - }) = candidate.rollup.asap() - else { - unreachable!("a sketch rollup candidate's rollup is a regrouping merge"); - }; - // A compensating filter is kept over the resized states. - let states = match candidate.rollup.children()[0].non_asap() { - Some(NonASAPOp::Filter { pred, .. }) => { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: pred.clone(), - child: Rc::clone(&sized), - }))? - } - _ => Rc::clone(&sized), - }; - let merge = regroup_merge(&states, keys.clone(), &candidate.coarse)?; - replaced.push((Rc::as_ptr(&candidate.coarse), merge)); - } - } - Ok(replaced) -} - -/// Every non-empty subset of `candidates`, in binary-counter order (the -/// first candidate the lowest bit); an error past [`MAX_REUSE_CANDIDATES`]. -fn subsets(candidates: &[T], rule: &'static str) -> Result>, CseError> { - let n = candidates.len(); - if n > MAX_REUSE_CANDIDATES { - return Err(CseError::TooManyCandidates { - counted: match rule { - "rollup" => "rollup choices", - _ => "top-k reuse candidates", - }, - found: n, - max: MAX_REUSE_CANDIDATES, - }); - } - Ok((1u64..(1u64 << n)) - .map(|mask| { - (0..n) - .filter(|&i| mask >> i & 1 == 1) - .map(|i| candidates[i].clone()) - .collect() - }) - .collect()) -} - -/// `roots` with every node in `replaced` swapped for its replacement, and -/// the nodes a replacement reads rebuilt too (a rollup may read a build -/// another rollup replaces); every guarantee composed again. -fn replace_all( - roots: &[QueryRoot], - replaced: &HashMap<*const OperatorNode, Rc>, -) -> Result, CseError> { - fn walk( - node: &Rc, - replaced: &HashMap<*const OperatorNode, Rc>, - rebuilt: &mut HashMap<*const OperatorNode, Rc>, - ) -> Result, CseError> { - if let Some(done) = rebuilt.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(done)); - } - let source = replaced.get(&Rc::as_ptr(node)).unwrap_or(node); - let mut changed = !Rc::ptr_eq(source, node); - let mut children = HashMap::new(); - for child in source.children() { - let new = walk(child, replaced, rebuilt)?; - changed |= !Rc::ptr_eq(&new, child); - children.insert(Rc::as_ptr(child), new); - } - let out = if !changed { - Rc::clone(node) - } else if children - .iter() - .all(|(old, new)| std::ptr::eq(*old, Rc::as_ptr(new))) - { - Rc::clone(source) - } else { - Rc::new(source.with_new_children(|child| children[&Rc::as_ptr(child)].clone())?) - }; - rebuilt.insert(Rc::as_ptr(node), Rc::clone(&out)); - Ok(out) - } - let mut rebuilt = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => QueryRoot::Operator(walk(node, replaced, &mut rebuilt)?), - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), walk(node, replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// Exact top-k computations of one input and partition with at least two -/// different `k`, in the order a children-first walk meets them, and the -/// ranking they all read in a reuse candidate: `Limit(k_max)` over `Sort` -/// of the input, partitioned as they are. -#[derive(Debug, Clone)] -pub(crate) struct TopKReuseCandidate { - /// The plain `Aggregate` top-k nodes of one input and partition, in - /// children-first order; at least two different `k`. - pub members: Vec>, - /// `Limit(k_max)` over a descending `Sort` of their input, partitioned - /// as they are: what every member reads in a reuse candidate. - pub ranking: Rc, -} - -/// The exact top-k reuse candidates of the candidate with query roots `roots`: -/// plain (exact) `Aggregate` top-k nodes over an exact input, unfiltered and -/// grouped by listed keys, grouped by equal input and partition. -pub(crate) fn topk_reuse_candidates(roots: &[QueryRoot]) -> Vec { - let mut nodes = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let mut groups: Vec>> = Vec::new(); - for node in nodes.into_iter().filter(|n| exact_topk(n).is_some()) { - let same = |member: &Rc| { - let (Some((child, by, _)), Some((other, other_by, _))) = - (exact_topk(member), exact_topk(&node)) - else { - return false; - }; - by == other_by && (Rc::ptr_eq(child, other) || child == other) - }; - match groups.iter_mut().find(|g| same(&g[0])) { - Some(group) => group.push(node), - None => groups.push(vec![node]), - } - } - groups - .into_iter() - .filter_map(|members| { - let ks: Vec = members - .iter() - .filter_map(|m| exact_topk(m)) - .map(|t| t.2) - .collect(); - let largest = *ks.iter().max()?; - if ks.iter().all(|&k| k == largest) { - return None; - } - let (child, by, _) = exact_topk(&members[0])?; - let ranking = ranking(child, by, largest)?; - Some(TopKReuseCandidate { members, ranking }) - }) - .collect() -} - -/// The input, partition and `k` of an exact top-k over an exact input. -fn exact_topk(node: &OperatorNode) -> Option<(&Rc, &Reduction, usize)> { - let Some(NonASAPOp::Aggregate { - child, - reduction: reduction @ Reduction::Reduce(by), - measures, - filters, - having: None, - .. - }) = node.non_asap() - else { - return None; - }; - let [AggIntent::TopK { k, .. }] = measures.as_slice() else { - return None; - }; - let exact = child - .guarantee - .as_ref() - .is_some_and(asap_types::ir::properties::AccuracyGuarantee::is_exact); - (exact && !by.is_without() && filters.iter().all(Option::is_none)) - .then_some((child, reduction, *k)) -} - -/// The `k` highest-ranked rows of `child` per partition: `Limit` over a -/// descending `Sort` by its ranked column, as exact top-k runs. -fn ranking(child: &Rc, by: &Reduction, k: usize) -> Option> { - let Reduction::Reduce(keys) = by else { - return None; - }; - let value = asap_types::ir::schema::topk::ranked_column(child).ok()?; - let sorted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(value), - ascending: false, - nulls_first: false, - }], - partition_by: keys.clone(), - child: Rc::clone(child), - })) - .ok()?; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Limit { - n: Some(k), - offset: 0, - partition_by: keys.clone(), - child: sorted, - })) - .ok() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_support::lower_promql; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::schema::schema_support::with_promql_series_identity; - use asap_types::ir::schema::SketchAlgorithm; - - fn roots(queries: &[&str]) -> Vec { - queries - .iter() - .map(|q| { - let root = lower_promql(q, AccuracyTarget::exact()); - QueryRoot::Operator(with_promql_series_identity(&root).unwrap()) - }) - .collect() - } - - /// The positions in `all` of the items of `chosen`, matched by the node - /// `node` gives for each. - fn positions(all: &[T], chosen: &[T], node: impl Fn(&T) -> &Rc) -> Vec { - chosen - .iter() - .map(|item| { - all.iter() - .position(|other| Rc::ptr_eq(node(other), node(item))) - .expect("a chosen item is listed") - }) - .collect() - } - - /// `sum by (job) (m)` and `max by (host, job) (m)` scan `m` with other - /// labels; widened to `job, host`, both scans project the same - /// columns, and each root keeps its columns. - #[test] - fn scans_of_one_metric_with_other_labels_share_once_widened() { - let roots = roots(&["sum by (job) (m)", "max by (host, job) (m)"]); - let widened = with_union_scan_labels(&roots).unwrap().unwrap(); - let scan_labels = |root: &QueryRoot| { - let QueryRoot::Operator(node) = root else { - unreachable!() - }; - OperatorNode::reachable(node) - .iter() - .find_map(|n| match n.non_asap() { - Some(NonASAPOp::Scan { schema, .. }) => Some( - schema - .fields - .iter() - .map(|f| f.name.clone()) - .collect::>(), - ), - _ => None, - }) - .unwrap() - }; - assert_eq!( - scan_labels(&widened[0]), - ["ts", "value", "job", "host", PROMQL_SERIES_IDENTITY] - ); - assert_eq!(scan_labels(&widened[0]), scan_labels(&widened[1])); - // The unique key still names the series identity and ts. - let QueryRoot::Operator(node) = &widened[0] else { - unreachable!() - }; - let scan = OperatorNode::reachable(node) - .into_iter() - .find(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .unwrap(); - assert_eq!(scan.schema.unique_keys, [vec![4, 0]]); - for (root, wide) in roots.iter().zip(&widened) { - let (QueryRoot::Operator(root), QueryRoot::Operator(wide)) = (root, wide) else { - unreachable!() - }; - assert_eq!(root.schema, wide.schema); - } - // Scans that already agree are left alone. - assert!(with_union_scan_labels(&widened).unwrap().is_none()); - } - - /// A count over a window, read from a Count-Min built in tumbling - /// tumbling windows, keeps the guarantee of one build over the window: Pass 1 - /// offers Count-Min for counts, and its merged counters add. - #[test] - fn a_count_from_count_min_tumbling_windows_keeps_its_guarantee() { - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let root = lower_promql( - "count_over_time(m[5m])", - asap_types::ir::properties::AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&[QueryRoot::Operator( - with_promql_series_identity(&root).unwrap(), - )]), - ) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(|c| { - *c == asap_types::stages::HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::Cms, - ) - }) - .expect("Count-Min is offered for a count") - }) - .collect(); - let roots: Vec = compose_logical_candidate(&inventory, &choice).unwrap(); - let candidates = tumbling_window_candidates(&roots, &[Some(60_000)]); - let pass2_candidates = window_composition_pass2_candidates(&roots, &candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate: {pass2_candidates:?}") - }; - let (QueryRoot::Operator(whole), QueryRoot::Operator(windowed)) = - (&roots[0], &pass2_candidate.roots.as_ref().unwrap()[0]) - else { - panic!("operator roots") - }; - assert!(whole.guarantee.is_some()); - assert_eq!(windowed.guarantee, whole.guarantee); - } - - /// The composed candidate of `queries` (each at its own ε, δ = 0.01) - /// in which every target chose `family`. - fn chose(queries: &[(&str, f64)], family: SketchAlgorithm) -> Vec { - use asap_types::stages::HowToComputeAggregateMeasure; - chose_where(queries, |c| { - *c == HowToComputeAggregateMeasure::Summary(family) - }) - } - - /// The composed candidate of `queries` (each at its own ε, δ = 0.01) - /// in which every target chose the first candidate `pick` accepts. - fn chose_where( - queries: &[(&str, f64)], - pick: impl Fn(&asap_types::stages::HowToComputeAggregateMeasure) -> bool, - ) -> Vec { - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let roots: Vec<_> = queries - .iter() - .map(|(q, epsilon)| { - let root = lower_promql( - q, - asap_types::ir::properties::AccuracyTarget::max_error(*epsilon) - .failure_probability(0.01) - .accuracy_metric(asap_types::ir::properties::AccuracyMetric::Rank), - ); - QueryRoot::Operator(with_promql_series_identity(&root).unwrap()) - }) - .collect(); - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(&pick) - .expect("the family is a candidate") - }) - .collect(); - compose_logical_candidate(&inventory, &choice).unwrap() - } - - /// The summary builds `roots` read, by address, with each one's KLL `k`. - fn kll_builds(roots: &[QueryRoot]) -> Vec<(*const OperatorNode, u32)> { - let mut out: Vec<(*const OperatorNode, u32)> = Vec::new(); - for node in roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - { - if let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = node.asap() - { - let asap_types::ir::schema::SketchParams::Kll { k, .. } = kind.params() else { - panic!("{kind:?}") - }; - if !out.iter().any(|(p, _)| *p == Rc::as_ptr(&node)) { - out.push((Rc::as_ptr(&node), *k)); - } - } - } - out - } - - fn root_bound(root: &QueryRoot) -> f64 { - let QueryRoot::Operator(node) = root else { - panic!("operator root") - }; - node.guarantee - .as_ref() - .and_then(|g| g.spec.error.evaluate()) - .expect("a constant bound") - } - - /// The tumbling-window candidates of `roots` as (tumbling-window width, tumbling-window count). - fn tumbling_windows(roots: &[QueryRoot], interval_ms: &[Option]) -> Vec<(u64, u32)> { - tumbling_window_candidates(roots, interval_ms) - .iter() - .map(|rule_candidate| { - ( - rule_candidate.tumbling_window_width_ms, - rule_candidate.tumbling_window_count, - ) - }) - .collect() - } - - /// A 5-min window evaluated every 2 min, 90 s or 1 min is built in tumbling windows - /// of gcd(window, interval). A once-per-window reader has one tumbling window, - /// which can reuse the newest tumbling window of a longer chain. - #[test] - fn tumbling_window_width_is_the_gcd_of_the_window_and_the_interval() { - let roots = chose( - &[("quantile_over_time(0.99, m[5m])", 0.01)], - SketchAlgorithm::Kll, - ); - assert_eq!(tumbling_windows(&roots, &[Some(120_000)]), [(60_000, 5)]); - assert_eq!(tumbling_windows(&roots, &[Some(90_000)]), [(30_000, 10)]); - assert_eq!(tumbling_windows(&roots, &[Some(60_000)]), [(60_000, 5)]); - assert_eq!(tumbling_windows(&roots, &[Some(300_000)]), [(300_000, 1)]); - assert!(tumbling_windows(&roots, &[None]).is_empty()); - } - - /// Past their caps, the tumbling-window rule and the reuse subsets name how many - /// candidates they found. - #[test] - fn too_many_tumbling_window_or_reuse_candidates_name_the_count_found() { - let roots = chose( - &[("quantile_over_time(0.99, m[5m])", 0.01)], - SketchAlgorithm::Kll, - ); - let tumbling_windows: Vec<_> = tumbling_window_candidates(&roots, &[Some(60_000)]) - .into_iter() - .cycle() - .take(MAX_TUMBLING_WINDOW_CANDIDATES + 1) - .collect(); - let error = window_composition_pass2_candidates(&roots, &tumbling_windows).unwrap_err(); - assert_eq!( - error.to_string(), - format!( - "found {} summary builds that could be built in tumbling windows, more than \ - {MAX_TUMBLING_WINDOW_CANDIDATES}", - MAX_TUMBLING_WINDOW_CANDIDATES + 1 - ) - ); - let error = subsets(&[0u8; MAX_REUSE_CANDIDATES + 1], "rollup").unwrap_err(); - assert_eq!( - error.to_string(), - format!( - "found {} rollup choices, more than {MAX_REUSE_CANDIDATES}", - MAX_REUSE_CANDIDATES + 1 - ) - ); - } - - /// A range read at an `@` anchor has no tumbling windows, however many shifts - /// stack above the anchored one: `m[5m] @ 100` with an extra - /// `offset 1m` shift between the range and the anchor is still anchored. - #[test] - fn an_anchor_below_stacked_shifts_has_no_tumbling_windows() { - use asap_types::ir::operator::TimeShift; - let roots = chose( - &[("quantile_over_time(0.99, m[5m] @ 100)", 0.01)], - SketchAlgorithm::Kll, - ); - let QueryRoot::Operator(root) = &roots[0] else { - panic!("operator root") - }; - let build = OperatorNode::reachable(root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("a build"); - assert_eq!(splittable_window_ms(&build), None, "one anchored shift"); - // Put an `offset 1m` shift between the range and its anchored shift. - let restacked = build - .with_new_children(|range| { - Rc::new( - range - .with_new_children(|anchored| { - assert!(matches!( - anchored.non_asap(), - Some(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_some() - )); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 60_000, - at: None, - }, - child: anchored.clone(), - })) - .unwrap() - }) - .unwrap(), - ) - }) - .unwrap(); - assert_eq!( - splittable_window_ms(&restacked), - None, - "anchor below an offset" - ); - } - - /// Exact accumulators whose states merge get tumbling windows; IRate, which reads - /// the last two samples of the window, does not. - #[test] - fn exact_accumulators_that_merge_get_tumbling_windows() { - use asap_types::stages::HowToComputeAggregateMeasure; - let accumulator = |c: &HowToComputeAggregateMeasure| { - matches!(c, HowToComputeAggregateMeasure::ExactAccumulator(_)) - }; - for query in [ - "rate(m[5m])", - "increase(m[5m])", - "sum_over_time(m[5m])", - "max_over_time(m[5m])", - ] { - let roots = chose_where(&[(query, 0.01)], accumulator); - assert_eq!( - tumbling_windows(&roots, &[Some(60_000)]), - [(60_000, 5)], - "{query}" - ); - } - let roots = chose_where(&[("irate(m[5m])", 0.01)], accumulator); - assert!(tumbling_windows(&roots, &[Some(60_000)]).is_empty()); - } - - /// The rule's candidate reads a `SummaryMerge` over one `SummaryTumblingWindowChain` of - /// the build's family and size over the range's input, keeps the - /// build's output names, and keeps the whole build's guarantee (a KLL - /// merges into the state of one build over the union). - #[test] - fn tumbling_windows_replace_the_build_with_a_merged_chain() { - let roots = chose( - &[("quantile_over_time(0.99, m[5m])", 0.01)], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(60_000)]); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate: {pass2_candidates:?}") - }; - assert_eq!( - positions( - &rule_candidates, - &pass2_candidate.tumbling_window_candidates, - |c| &c.build - ), - [0] - ); - let tumbling_windows = pass2_candidate.roots.as_ref().unwrap(); - let QueryRoot::Operator(root) = &tumbling_windows[0] else { - panic!("operator root") - }; - root.validate_structure().unwrap(); - let QueryRoot::Operator(whole) = &roots[0] else { - panic!("operator root") - }; - assert_eq!(root.guarantee, whole.guarantee); - assert!(root.guarantee.is_some()); - let merge = OperatorNode::reachable(root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryMerge { .. }))) - .expect("a merge"); - let Some(ASAPOp::SummaryMerge { children, .. }) = merge.asap() else { - unreachable!() - }; - let [chain] = children.as_slice() else { - panic!("one chain") - }; - let Some(ASAPOp::SummaryTumblingWindowChain { - child, - family, - tumbling_window_width_ms: 60_000, - tumbling_window_count: 5, - .. - }) = chain.asap() - else { - panic!("{:?}", chain.operator) - }; - let Some(ASAPOp::SummaryAgg { - child: range, - family: whole_family, - .. - }) = rule_candidates[0].build.asap() - else { - unreachable!() - }; - assert_eq!(family, whole_family); - assert_eq!(child.operator, range.children()[0].operator); - assert_eq!(chain.schema, rule_candidates[0].build.schema); - assert_eq!(merge.coverage(), rule_candidates[0].build.coverage()); - } - - /// Two builds that could be split give three candidates: each alone and - /// both, in binary-counter order. Their windows (5 and 2 min) nest on - /// 1-min tumbling windows, so both together are also offered as one shared chain. - #[test] - fn every_non_empty_subset_of_the_tumbling_window_candidates_is_a_pass2_candidate() { - let roots = chose( - &[ - ("quantile_over_time(0.99, m[5m])", 0.01), - ("quantile_over_time(0.5, m[2m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(60_000), Some(60_000)]); - assert_eq!(rule_candidates.len(), 2); - let subsets: Vec<_> = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .iter() - .map(|v| { - assert!(v.roots.is_ok()); - ( - positions(&rule_candidates, &v.tumbling_window_candidates, |c| { - &c.build - }), - v.shared_chains.clone(), - ) - }) - .collect(); - assert_eq!( - subsets, - [ - (vec![0], vec![]), - (vec![1], vec![]), - (vec![0, 1], vec![]), - ( - vec![0, 1], - vec![SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 60_000, - tumbling_window_count: 5 - }] - ) - ] - ); - } - - /// The tumbling-window chains in `roots`, one per distinct node. - fn chains(roots: &[QueryRoot]) -> Vec> { - let mut found: Vec> = Vec::new(); - for root in roots { - for node in OperatorNode::reachable(operator(root)) { - if matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - && !found.iter().any(|c| Rc::ptr_eq(c, &node)) - { - found.push(node); - } - } - } - found - } - - /// The `newest_tumbling_windows` of the one merge `root` reads. - fn newest_tumbling_windows(root: &QueryRoot) -> Option { - OperatorNode::reachable(operator(root)) - .into_iter() - .find_map(|n| match n.asap() { - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - }) => Some(*newest_tumbling_windows), - _ => None, - }) - .expect("a merge") - } - - /// A 1-min and a 5-min KLL quantile over one metric, both every 10 s, - /// are 6 and 30 tumbling windows of 10 s; the 1-min window is the newest 6 of the - /// 5-min one. Besides each query's own chain, the rule offers one chain - /// of 30 tumbling windows read by both: the 5-min query merges every tumbling window, the - /// 1-min one only the newest 6, and each keeps the guarantee of its own - /// whole-window build (a KLL merges exactly). - #[test] - fn nested_windows_on_equal_tumbling_windows_share_one_chain() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[1m])", 0.01), - ("quantile_over_time(0.9, m[5m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(10_000), Some(10_000)]); - assert_eq!( - tumbling_windows(&roots, &[Some(10_000), Some(10_000)]), - [(10_000, 6), (10_000, 30)] - ); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let shared: Vec<_> = pass2_candidates - .iter() - .filter(|c| !c.shared_chains.is_empty()) - .collect(); - let [shared] = shared.as_slice() else { - panic!("one shared-chain candidate: {pass2_candidates:?}") - }; - assert_eq!( - shared.shared_chains, - [SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 10_000, - tumbling_window_count: 30 - }] - ); - let shared_roots = shared.roots.as_ref().unwrap(); - let [chain] = chains(shared_roots) - .try_into() - .unwrap_or_else(|c: Vec<_>| panic!("one chain, found {}", c.len())); - assert!(matches!( - chain.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 10_000, - tumbling_window_count: 30, - .. - }) - )); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), Some(6)); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), None); - for (root, whole) in shared_roots.iter().zip(&roots) { - let root = operator(root); - root.validate_structure().unwrap(); - assert!(root.guarantee.is_some()); - assert_eq!(root.guarantee, operator(whole).guarantee); - } - // The per-query chains stay candidates: two chains, each read in full. - let separate = pass2_candidates - .iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty()) - .expect("both builds in their own tumbling windows"); - assert_eq!(chains(separate.roots.as_ref().unwrap()).len(), 2); - } - - /// A 15-min quantile every minute and the same quantile every 5 min - /// are 15 × 1-min and 3 × 5-min tumbling windows. The rule also offers one chain - /// at the finer width, 15 × 1 min, that both read in full, each at its - /// own interval; a 1-hour quantile every 5 min (12 × 5 min) reads the - /// newest 60 of one 60 × 1-min chain with them. - #[test] - fn queries_at_different_intervals_share_one_chain_at_the_finer_width() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[15m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(60_000), Some(300_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(60_000, 15), (300_000, 3)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let shared = pass2_candidates - .iter() - .find(|c| !c.shared_chains.is_empty()) - .expect("a shared-chain candidate"); - assert_eq!( - shared.shared_chains, - [SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 60_000, - tumbling_window_count: 15 - }] - ); - let shared_roots = shared.roots.as_ref().unwrap(); - let [chain] = chains(shared_roots) - .try_into() - .unwrap_or_else(|c: Vec<_>| panic!("one chain, found {}", c.len())); - assert!(matches!( - chain.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 15, - .. - }) - )); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), None); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), None); - for (root, whole) in shared_roots.iter().zip(&roots) { - operator(root).validate_structure().unwrap(); - assert_eq!(operator(root).guarantee, operator(whole).guarantee); - } - - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[1h])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| !c.shared_chains.is_empty()) - .expect("a shared-chain candidate"); - let shared_roots = shared.roots.unwrap(); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), Some(15)); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), None); - assert!(matches!( - chains(&shared_roots)[0].asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 60, - .. - }) - )); - } - - /// A build whose tumbling windows the group's finest tumbling windows do not divide gets its - /// own chain without breaking the group: 20-min and 10-min quantiles - /// every 2 min (2-min tumbling windows) share one chain, and a 15-min quantile - /// every 3 min (3-min tumbling windows) stays apart. - #[test] - fn a_build_on_an_incompatible_grid_does_not_break_a_shared_chain() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[20m])", 0.01), - ("quantile_over_time(0.9, m[10m])", 0.01), - ("quantile_over_time(0.99, m[15m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(120_000), Some(120_000), Some(180_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(120_000, 10), (120_000, 5), (180_000, 5)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 3 && !c.shared_chains.is_empty()) - .expect("a shared chain among all three builds in tumbling windows"); - assert_eq!( - shared.shared_chains, - [SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 120_000, - tumbling_window_count: 10 - }] - ); - assert_eq!(chains(shared.roots.as_ref().unwrap()).len(), 2); - } - - /// The shared chains (width, count) of every Pass 2 candidate that - /// windowed all of `rule_candidates`. - fn shared_grids( - roots: &[QueryRoot], - rule_candidates: &[TumblingWindowCandidate], - ) -> Vec> { - window_composition_pass2_candidates(roots, rule_candidates) - .unwrap() - .into_iter() - .filter(|c| c.tumbling_window_candidates.len() == rule_candidates.len()) - .map(|c| { - c.shared_chains - .iter() - .map(|shared| { - ( - shared.tumbling_window_width_ms, - shared.tumbling_window_count, - ) - }) - .collect() - }) - .collect() - } - - /// Tumbling windows neither member's divide (3-min and 5-min tumbling windows of a 15-min - /// window) are shared at their gcd, 1 min, as one more candidate beside - /// the per-query chains; no member's own width serves both. - #[test] - fn a_shared_grid_at_the_gcd_of_the_members_widths_is_offered() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[15m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(180_000), Some(300_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(180_000, 5), (300_000, 3)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - assert_eq!( - shared_grids(&roots, &rule_candidates), - [vec![], vec![(60_000, 15)]] - ); - } - - /// A 15-min quantile every minute (15 × 1-min tumbling windows) and a 6-min one - /// every 90 s (4 × 90-s tumbling windows): one chain of 30-s tumbling windows over 15 min - /// serves both, the 6-min query merging the newest 12. It comes beside - /// the per-query chains, and Stage 3 prices which is cheaper. - #[test] - fn queries_at_60_and_90_seconds_share_one_chain_of_30_second_tumbling_windows() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[6m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(60_000), Some(90_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(60_000, 15), (90_000, 4)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - assert_eq!( - shared_grids(&roots, &rule_candidates), - [vec![], vec![(30_000, 30)]] - ); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| !c.shared_chains.is_empty()) - .unwrap(); - let shared_roots = shared.roots.unwrap(); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), None); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), Some(12)); - for (root, whole) in shared_roots.iter().zip(&roots) { - operator(root).validate_structure().unwrap(); - assert_eq!(operator(root).guarantee, operator(whole).guarantee); - } - } - - /// When the members' finest width divides the others, the gcd is that - /// width: one shared candidate, not two equal ones. With 1-min, 2-min - /// and 3-min tumbling windows, the divisible group (1 and 2 min, the 3-min build - /// joining at 1 min too) is already the gcd grid. - #[test] - fn the_gcd_grid_is_not_listed_twice_when_a_member_width_is_the_gcd() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[12m])", 0.01), - ("quantile_over_time(0.9, m[12m])", 0.01), - ("quantile_over_time(0.99, m[12m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(60_000), Some(120_000), Some(180_000)]; - let rule_candidates = tumbling_window_candidates(&roots, &grids); - assert_eq!( - shared_grids(&roots, &rule_candidates), - [vec![], vec![(60_000, 12)]] - ); - } - - /// The shared chain is sized for its strictest reader - /// ([`size_for_all_readers`]): a 1-min quantile at ε = 0.01 and a - /// 5-min one at ε = 0.05 read one chain of the 1-min query's `k`. - #[test] - fn a_shared_chain_is_sized_for_its_strictest_reader() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[1m])", 0.01), - ("quantile_over_time(0.9, m[5m])", 0.05), - ], - SketchAlgorithm::Kll, - ); - let family = |node: &OperatorNode| match node.asap() { - Some( - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => family.clone(), - _ => unreachable!(), - }; - let strict = family(&builds(&roots[0])[0]); - assert_ne!(strict, family(&builds(&roots[1])[0])); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(10_000), Some(10_000)]); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let shared = pass2_candidates - .iter() - .find(|c| !c.shared_chains.is_empty()) - .expect("a shared-chain candidate"); - let shared_roots = shared.roots.as_ref().unwrap(); - let [chain] = chains(shared_roots) - .try_into() - .unwrap_or_else(|c: Vec<_>| panic!("one chain, found {}", c.len())); - assert_eq!(family(&chain), strict); - // The 1-min query keeps its own guarantee; the 5-min one reads the - // stricter state, so its guarantee is the 1-min build's size's, - // not its own looser one. - let guarantee = |root: &QueryRoot| operator(root).guarantee.clone().unwrap(); - assert_eq!(guarantee(&shared_roots[0]), guarantee(&roots[0])); - assert_ne!(guarantee(&shared_roots[1]), guarantee(&roots[1])); - assert_eq!( - guarantee(&shared_roots[1]).spec, - guarantee(&shared_roots[0]).spec - ); - } - /// The composed candidate of `queries` with every target at its exact - /// option (an exact accumulator, or plain exact execution). - fn exact(queries: &[&str]) -> Vec { - use asap_types::stages::HowToComputeAggregateMeasure as C; - let queries: Vec<(&str, f64)> = queries.iter().map(|q| (*q, 0.01)).collect(); - chose_where(&queries, |c| matches!(c, C::Exact | C::ExactAccumulator(_))) - } - - fn operator(root: &QueryRoot) -> &Rc { - let QueryRoot::Operator(node) = root else { - panic!("operator root") - }; - node - } - - fn builds(root: &QueryRoot) -> Vec> { - OperatorNode::reachable(operator(root)) - .into_iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect() - } - - /// One coarse build's rollup candidate, computed alone, is the one - /// [`rollup_candidates`] lists for it, and needs a finer build to exist. - #[test] - fn one_coarse_builds_rollup_candidate_is_the_listed_one() { - let roots = exact(&[ - "sum by (job) (m)", - "sum by (job, instance) (m)", - "sum by (job, instance, zone) (m)", - ]); - let listed = rollup_candidates(&roots); - assert!(!listed.is_empty()); - for candidate in &listed { - let alone = rollup_candidate_for(&roots, Rc::as_ptr(&candidate.coarse)).unwrap(); - assert!(Rc::ptr_eq(&alone.finer, &candidate.finer)); - } - let coarse = &builds(&roots[0])[0]; - assert!(rollup_candidate_for(&roots[..1], Rc::as_ptr(coarse)).is_none()); - } - - /// `sum by (job)` is answered from `sum by (job, instance)`, which the - /// workload computes anyway: the coarse query reads the finer query's - /// finalized sums, summed by `job`, so it builds nothing over the raw - /// rows; both values stay exact and keep their columns. - #[test] - fn rollup_from_a_finer_exact_sum() { - let roots = exact(&["sum by (job) (m)", "sum by (job, instance) (m)"]); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - assert!(Rc::ptr_eq(&rule_candidate.coarse, &builds(&roots[0])[0])); - let pass2_candidates = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate") - }; - let rolled = pass2_candidate.roots.as_ref().unwrap(); - let coarse = builds(&rolled[0]); - let fine = builds(&rolled[1]); - assert_eq!(fine.len(), 1); - // The finer build, then the sum over its values. - assert_eq!(coarse.len(), 2); - assert!(Rc::ptr_eq(&coarse[0], &fine[0]) || Rc::ptr_eq(&coarse[1], &fine[0])); - let rollup = coarse.iter().find(|b| !Rc::ptr_eq(b, &fine[0])).unwrap(); - let Some(ASAPOp::SummaryAgg { child, family, .. }) = rollup.asap() else { - unreachable!() - }; - assert!(matches!( - child.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - )); - assert_eq!( - *family, - FieldDataType::ExactAggregate(ExactKind::Sum, asap_types::ir::schema::ExactParams::Sum) - ); - for (before, after) in roots.iter().zip(rolled) { - let (before, after) = (operator(before), operator(after)); - after.validate_structure().unwrap(); - assert_eq!(before.schema, after.schema); - assert!(after - .guarantee - .as_ref() - .is_some_and(asap_types::ir::properties::AccuracyGuarantee::is_exact)); - } - } - - /// Identical per-series averages retain their values under label projection; - /// different windows, functions, and source predicates must stay separate. - #[test] - fn rollup_crosses_only_identical_per_series_reductions() { - let input = |query: &str| { - let root = lower_promql(query, asap_types::ir::properties::AccuracyTarget::exact()); - OperatorNode::reachable(&root) - .into_iter() - .find(|node| { - matches!( - node.non_asap(), - Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - .. - }) - ) - }) - .unwrap() - }; - let fine = input("sum by (job, instance) (avg_over_time(m[1m]))"); - assert!(same_rows( - &input("sum by (job) (avg_over_time(m[1m]))"), - &fine - )); - for coarse in [ - "sum by (job) (avg_over_time(m[2m]))", - "sum by (job) (max_over_time(m[1m]))", - "sum by (job) (avg_over_time(m{zone=\"east\"}[1m]))", - ] { - assert!(!same_rows(&input(coarse), &fine), "{coarse}"); - } - } - - /// Minima and maxima roll up as extrema; counts merge their integer states. - #[test] - fn extrema_and_integer_counts_roll_up() { - assert!(!rollup_candidates(&exact(&[ - "count by (job) (m)", - "count by (job, instance) (m)" - ])) - .is_empty()); - for (query, kind) in [ - ("min by (job) (m)", ExactKind::Min), - ("max by (job) (m)", ExactKind::Max), - ] { - let finer = query.replace("(job)", "(job, instance)"); - let roots = exact(&[query, &finer]); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("{query}: one candidate: {rule_candidates:?}") - }; - let mut value = Rc::clone(&rule_candidate.rollup); - if let Some(NonASAPOp::Project { child, .. }) = value.non_asap() { - value = Rc::clone(child); - } - let Some(ASAPOp::FinalizeExactAccumulator { child: state }) = value.asap() else { - panic!("a finalized value") - }; - let Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(rolled, _), - .. - }) = state.asap() - else { - panic!("an exact accumulator") - }; - assert_eq!(*rolled, kind, "{query}"); - let pass2_candidate = rollup_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .remove(0); - assert_eq!( - operator(&pass2_candidate.roots.unwrap()[0]).schema, - operator(&roots[0]).schema, - "{query}" - ); - } - } - - /// The rule reuses only a finer computation the workload already has: - /// keys that are not a superset, another metric, window or measure, or - /// a lone query, give no tumbling-window candidate. - #[test] - fn rollup_needs_a_finer_computation_of_the_same_rows() { - for queries in [ - &["sum by (job) (m)"][..], - &["sum by (job) (m)", "sum by (instance) (m)"], - &["sum by (job) (m)", "sum by (job, instance) (n)"], - &["sum by (job) (m)", "max by (job, instance) (m)"], - &[ - "sum by (job) (sum_over_time(m[5m]))", - "sum by (job, instance) (sum_over_time(m[1m]))", - ], - &["sum by (job) (m)", "sum by (job) (m)"], - &["sum by (job) (m)", "sum without (instance) (m)"], - &["sum without (job) (m)", "sum by (job, instance) (m)"], - ] { - assert!(rollup_candidates(&exact(queries)).is_empty(), "{queries:?}"); - } - } - - /// Rollup follows the family-capability table: sums, counts and - /// extrema merge exactly across groups; counter states merge only in - /// time, IRate not at all; a heap sketch's merged top-k has no - /// guarantee, and no sketch's states can be regrouped in the IR. - #[test] - fn rollup_is_refused_for_families_without_an_exact_merge_across_groups() { - use asap_types::ir::schema::{ExactParams as P, SketchParams}; - let exact = |kind, params| FieldDataType::ExactAggregate(kind, params); - assert_eq!( - rollup_kind(&exact(ExactKind::Sum, P::Sum)), - Some(ExactKind::Sum) - ); - assert_eq!( - rollup_kind(&exact(ExactKind::Count, P::Count)), - Some(ExactKind::Sum) - ); - for family in [ - exact(ExactKind::Rate, P::Rate), - exact(ExactKind::Increase, P::Increase), - exact(ExactKind::IRate, P::IRate), - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 5, - heap_size: 10, - }, - ), - GroupingStrategy::default(), - ), - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ), - ] { - assert_eq!(rollup_kind(&family), None, "{family:?}"); - } - } - - /// #840's `quantile by (host, job)` and `quantile by (job)`, both KLL, - /// the coarse one at a tighter ε: the coarse query reads the regrouping - /// merge of the finer KLL's states by `job`, so one KLL is built, sized - /// for its strictest reader (the larger `k`), and the coarse query keeps - /// a guarantee within its own target. - #[test] - fn a_coarse_sketch_rolls_up_from_the_finer_build() { - let roots = chose( - &[ - ("quantile by (host, job) (0.5, data)", 0.01), - ("quantile by (job) (0.5, data)", 0.001), - ], - SketchAlgorithm::Kll, - ); - let ks: Vec = kll_builds(&roots).into_iter().map(|(_, k)| k).collect(); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - assert!(Rc::ptr_eq(&rule_candidate.coarse, &builds(&roots[1])[0])); - assert!(Rc::ptr_eq(&rule_candidate.finer, &builds(&roots[0])[0])); - let pass2_candidates = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate") - }; - let rolled = pass2_candidate.roots.as_ref().unwrap(); - // One KLL, at the larger k, read by both queries. - assert_eq!( - kll_builds(rolled) - .into_iter() - .map(|(_, k)| k) - .collect::>(), - [*ks.iter().max().unwrap()] - ); - let merge = OperatorNode::reachable(operator(&rolled[1])) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryMerge { .. }))) - .expect("the coarse query reads a rollup"); - let Some(ASAPOp::SummaryMerge { - children, group_by, .. - }) = merge.asap() - else { - unreachable!() - }; - assert_eq!(group_by, &Some(vec![1])); - assert!(Rc::ptr_eq(&children[0], &builds(&rolled[0])[0])); - for (before, after) in roots.iter().zip(rolled) { - let (before, after) = (operator(before), operator(after)); - after.validate_structure().unwrap(); - assert_eq!(before.schema, after.schema); - } - // Each query's guarantee is the shared KLL's, within its target. - assert!( - root_bound(&rolled[1]) <= 0.001, - "{}", - root_bound(&rolled[1]) - ); - assert!(root_bound(&rolled[0]) <= 0.001); - } - - /// Every coarser grouping of one finest build is one choice: two coarse - /// builds of a three-grouping chain give one Pass 2 candidate, in which - /// both read the finest build's states. A finer build that is itself - /// coarse is not a source. - #[test] - fn coarse_groupings_of_one_finest_build_roll_up_together() { - let roots = chose( - &[ - ("quantile by (job) (0.9, m)", 0.01), - ("quantile by (job, instance) (0.9, m)", 0.01), - ("quantile by (job, instance, zone) (0.9, m)", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = rollup_candidates(&roots); - assert_eq!(rule_candidates.len(), 2); - let finest = &builds(&roots[2])[0]; - assert!(rule_candidates - .iter() - .all(|candidate| Rc::ptr_eq(&candidate.finer, finest))); - assert_eq!(rollup_choices(&rule_candidates).len(), 1); - let pass2_candidates = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate") - }; - let rolled = pass2_candidate.roots.as_ref().unwrap(); - assert_eq!(kll_builds(rolled).len(), 1); - } - - /// A quantile over every series rolls up from the KLLs by job: a - /// rollup to no keys. - #[test] - fn a_global_quantile_rolls_up_to_no_keys() { - let roots = chose( - &[ - ("quantile(0.9, m)", 0.01), - ("quantile by (job) (0.9, m)", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - assert!(matches!( - rule_candidate.rollup.asap(), - Some(ASAPOp::SummaryMerge { group_by: Some(keys), .. }) if keys.is_empty() - )); - let pass2 = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let rolled = pass2[0].roots.as_ref().unwrap(); - assert_eq!(kll_builds(rolled).len(), 1); - operator(&rolled[0]).validate_structure().unwrap(); - } - - /// Groupings that are not nested: `by (job)` rolls up from `by (job, - /// instance)` or `by (job, zone)`; the source is the one whose sorted - /// key names come first, whatever the query order. - #[test] - fn the_rollup_source_does_not_depend_on_query_order() { - for queries in [ - [ - "quantile by (job) (0.9, m)", - "quantile by (job, zone) (0.9, m)", - "quantile by (job, instance) (0.9, m)", - ], - [ - "quantile by (job) (0.9, m)", - "quantile by (job, instance) (0.9, m)", - "quantile by (job, zone) (0.9, m)", - ], - ] { - let roots = chose(&queries.map(|q| (q, 0.01)), SketchAlgorithm::Kll); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - let instance = queries.iter().position(|q| q.contains("instance")).unwrap(); - assert!(Rc::ptr_eq( - &rule_candidate.finer, - &builds(&roots[instance])[0] - )); - } - } - - /// A rollup needs the same rows: a coarse KLL over another metric is - /// none. - #[test] - fn sketch_rollup_needs_the_same_rows() { - let other_metric = chose( - &[ - ("quantile by (job) (0.9, n)", 0.01), - ("quantile by (job, instance) (0.9, m)", 0.01), - ], - SketchAlgorithm::Kll, - ); - assert!(rollup_candidates(&other_metric).is_empty()); - } - - /// Top-k over other partitions or inputs, or of one `k`, share no - /// ranking; a top-k a heap sketch answers is not reused. - #[test] - fn topk_reuse_needs_exact_topk_of_one_input_and_partition() { - for queries in [ - &[ - "topk(5, sum by (job) (m))", - "topk by (job) (10, sum by (job, instance) (m))", - ][..], - &["topk(5, sum by (job) (m))", "topk(10, sum by (job) (n))"], - &["topk(5, sum by (job) (m))", "topk(5, max by (job) (m))"], - &[ - "topk(5, sum by (job) (m))", - "topk by (job) (10, sum by (job) (m))", - ], - ] { - assert!( - topk_reuse_candidates(&exact(queries)).is_empty(), - "{queries:?}" - ); - } - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure as C; - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups( - &[ - "topk by (job) (5, sum_over_time(m[1m]))", - "topk by (job) (10, sum_over_time(m[1m]))", - ] - .iter() - .map(|q| { - let root = lower_promql( - q, - asap_types::ir::properties::AccuracyTarget::max_error(0.01) - .failure_probability(0.01), - ); - QueryRoot::Operator(root) - }) - .collect::>(), - ), - ) - .unwrap(); - // The CountSketch heap, the top-k's fused whole-expression one. - let heap = |c: &C| matches!(c, C::Summary(SketchAlgorithm::CountSketchWithHeap)); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - (0..t.choice_count()) - .position(|i| t.computation_for(i).is_some_and(|(c, _)| heap(c))) - .unwrap_or(0) - }) - .collect(); - let heaps: Vec = compose_logical_candidate(&inventory, &choice).unwrap(); - assert!(OperatorNode::reachable(operator(&heaps[0])) - .iter() - .any(|n| matches!(n.asap(), Some(ASAPOp::SummaryEstimate { .. })))); - assert!(topk_reuse_candidates(&heaps).is_empty()); - } -} diff --git a/crates/logical-asap-optimization/src/pass2/finer_aggregate.rs b/crates/logical-asap-optimization/src/pass2/finer_aggregate.rs deleted file mode 100644 index df857c35c..000000000 --- a/crates/logical-asap-optimization/src/pass2/finer_aggregate.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! Whether a query aggregate can be computed from a finer aggregate over the -//! same source and window, and with what on top (view matching for -//! aggregation queries, as in *Optimizing Queries Using Materialized Views: -//! A Practical, Scalable Solution*, SIGMOD 2001, §3.3, and Apache Calcite's -//! `SubstitutionVisitor.unifyAggregates`). -//! -//! The finer result holds at least the rows the query needs. A -//! *compensating* operation closes the gap: a compensating filter removes -//! the extra rows, and a rollup merges finer groups into the query's. A -//! filter that reads only finer group-by columns is constant within a finer -//! group, so filtering finer groups equals filtering input rows. -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; - -use asap_accuracy::propagation::{pinned_column, split_and}; - -/// How a query aggregate is computed from a finer one: keep the finer rows -/// that pass `compensating_filter`, then, when `needs_rollup`, merge them -/// into the query's groups. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct FinerAggregateMatch { - /// The query's conjuncts that the finer aggregate lacks, over the finer - /// group-by (column `i` is the finer aggregate's `i`-th key); `None` - /// when the finer aggregate has every conjunct. - pub compensating_filter: Option, - /// Each query group key, as an index into the finer group-by. - pub query_keys: Vec, - /// Whether finer groups merge into query groups: a finer key the query - /// drops is not pinned by an equality in `compensating_filter`. When - /// `false`, each query group is at most one finer row. - pub needs_rollup: bool, -} - -/// How a query aggregate over `query_rows` grouped by `query_keys` is -/// computed from a finer aggregate over `finer_rows` grouped by -/// `finer_keys` (keys are columns of the rows); `None` when it cannot be. -/// -/// Both rows must be the same scan (source and schema) under the same time -/// ranges; the finer conjuncts (scan predicates and filters) must be among -/// the query's; every other query conjunct must read only finer group-by -/// columns; and the query keys must be finer keys. -pub(crate) fn match_finer( - query_rows: &OperatorNode, - query_keys: &[usize], - finer_rows: &OperatorNode, - finer_keys: &[usize], -) -> Option { - let query = Rows::of(query_rows)?; - let finer = Rows::of(finer_rows)?; - if query.spine != finer.spine { - return None; - } - let mut extra = query.conjuncts; - for conjunct in finer.conjuncts { - let position = extra.iter().position(|other| *other == conjunct)?; - extra.remove(position); - } - let finer_key = |column: usize| finer_keys.iter().position(|&key| key == column); - let query_key_positions = query_keys - .iter() - .map(|&key| finer_key(key)) - .collect::>>()?; - if !extra.iter().all(|conjunct| { - conjunct.operator_refs().is_empty() - && conjunct - .columns_referenced() - .into_iter() - .all(|column| finer_key(column).is_some()) - }) { - return None; - } - let pinned: Vec = extra.iter().filter_map(pinned_column).collect(); - let needs_rollup = finer_keys - .iter() - .any(|key| !query_keys.contains(key) && !pinned.contains(key)); - // Without extra conjuncts or a rollup, the two aggregates have the same - // groups, perhaps in another key order: they are one computation, not - // one read from the other (each read from the other would be a cycle). - if extra.is_empty() && !needs_rollup { - return None; - } - let compensating_filter = (!extra.is_empty()).then(|| { - let remapped: Vec = extra - .iter() - .map(|conjunct| conjunct.map_columns(&mut |column| finer_key(column).unwrap())) - .collect(); - Predicate(match <[ScalarExpr; 1]>::try_from(remapped) { - Ok([one]) => one, - Err(many) => ScalarExpr::BoolAnd(many), - }) - }); - Some(FinerAggregateMatch { - compensating_filter, - query_keys: query_key_positions, - needs_rollup, - }) -} - -/// An aggregate's input rows: the scan and time ranges they come from, and -/// the conjuncts that select them. -struct Rows { - /// Each time range above the scan, outermost first, then the scan - /// without its predicates. - spine: Vec>, - /// Scan predicates and filter predicates, split at `AND`. - conjuncts: Vec, -} - -impl Rows { - /// The rows of `node`, a chain of `TimeRange`s and `Filter`s over a - /// `Scan`, none changing the scan's columns; `None` for anything else. - fn of(node: &OperatorNode) -> Option { - let mut rows = Rows { - spine: Vec::new(), - conjuncts: Vec::new(), - }; - let mut node = node; - loop { - match node.non_asap()? { - NonASAPOp::TimeRange { child, .. } if child.schema == node.schema => { - rows.spine.push(node.operator.map_children(|_| ())); - node = child; - } - NonASAPOp::Filter { pred, child } if child.schema == node.schema => { - split_and(&pred.0, &mut rows.conjuncts); - node = child; - } - NonASAPOp::Scan { - source, - schema, - predicates, - } => { - for predicate in predicates { - split_and(&predicate.0, &mut rows.conjuncts); - } - rows.spine.push(Operator::NonASAP(NonASAPOp::Scan { - source: source.clone(), - schema: schema.clone(), - predicates: Vec::new(), - })); - return Some(rows); - } - _ => return None, - } - } - } -} diff --git a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs b/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs deleted file mode 100644 index 678ba5c0d..000000000 --- a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! Stage 1, Pass 2's entry and output: the Pass 1 memo extended with the -//! candidates and groups that share work across computations -//! ([`asap_types::stages::LogicalASAPMemoCandidates`]), which Stage 2 implements. - -use crate::pass2::memo::rules::{ - finer_aggregate, rollup, scan_labels, shared_configuration_readers, summary_inputs, - summary_results, topk, -}; -use crate::pass2::memo::state_groups::{ - add_shared_summary_state_groups, check_unexpanded_memo, Pass2RuleError, -}; -use crate::pass2::memo::window_candidates::add_local_window_composition; -use asap_types::stages::Memo; -use asap_types::stages::{LogicalASAPMemoCandidates, PerComputationLogicalMemoCandidates}; -use asap_types::workload::QueryWorkloadEntry; - -/// Stage 1, Pass 2: adds to Pass 1's memo the local tumbling-window -/// candidates for every legal tumbling-window width (rule 1), then runs -/// rules 2–9 and `add_shared_summary_state_groups`. -/// `entries[i]` is query `i`'s workload entry. -pub fn generate_logical_asap_memo_candidates( - pass1: PerComputationLogicalMemoCandidates, - entries: &[QueryWorkloadEntry], -) -> Result { - pass2_after_local_window_composition(pass2_local_window_composition(pass1, entries)) -} - -/// Pass 2's rule 1 alone: Pass 1's memo with the local tumbling-window -/// candidates for every legal width (`asap_planner` prints this step). -pub fn pass2_local_window_composition( - pass1: PerComputationLogicalMemoCandidates, - entries: &[QueryWorkloadEntry], -) -> Memo { - let mut memo = pass1.memo; - // 1. Local window composition. - add_local_window_composition(&mut memo, entries); - memo -} - -/// The rest of Pass 2 on [`pass2_local_window_composition`]'s memo: rules -/// 2–9 and `add_shared_summary_state_groups`. -pub fn pass2_after_local_window_composition( - mut memo: Memo, -) -> Result { - apply_sharing_rules(&mut memo)?; - // 10. One group per shared summary build, read by its reader groups. - add_shared_summary_state_groups(&mut memo)?; - Ok(LogicalASAPMemoCandidates { memo }) -} - -/// Pass 2's rules 2–9, in order, on a memo after local window composition. -pub(crate) fn apply_sharing_rules(memo: &mut Memo) -> Result<(), Pass2RuleError> { - check_unexpanded_memo(memo)?; - // 2. Scan widening. - scan_labels::add_scan_label_candidates(memo)?; - // 3. Summary inputs. - summary_inputs::factor_summary_inputs(memo)?; - // 4. Top-k reuse. - topk::add_topk_candidates(memo)?; - // 5. Finer aggregate. - finer_aggregate::add_finer_aggregate_candidates(memo)?; - // 6. Rollup. - rollup::add_rollup_candidates(memo)?; - // 7. Window rollup. - rollup::add_window_rollup_candidates(memo)?; - // 8. Summary results. - summary_results::factor_summary_results(memo)?; - // 9. Shared configurations. - shared_configuration_readers::add_state_configuration_candidates(memo)?; - Ok(()) -} diff --git a/crates/logical-asap-optimization/src/pass2/memo.rs b/crates/logical-asap-optimization/src/pass2/memo.rs index c9bc66a80..754a06a5c 100644 --- a/crates/logical-asap-optimization/src/pass2/memo.rs +++ b/crates/logical-asap-optimization/src/pass2/memo.rs @@ -1,12 +1,6 @@ -//! The logical memo's candidates, rules and enumeration, over the memo -//! groups Stage 0 builds (`crate::stage0`). +//! The logical memo's rules over the memo groups Stage 0 builds +//! (`crate::stage0`). -/// The candidate DAGs a memo stands for: choices and their composition. -pub(crate) mod enumeration; pub(crate) mod rules; -/// Shared finite state configuration enumeration. -pub(crate) mod shared_configurations; -/// Concrete summary-state groups exposed by sharing rules. -pub(crate) mod state_groups; -/// Local tumbling-window candidates with selected-reader grid configurations. +/// Rule 1: tumbling-window layouts of a summary state group. pub(crate) mod window_candidates; diff --git a/crates/logical-asap-optimization/src/pass2/memo/enumeration.rs b/crates/logical-asap-optimization/src/pass2/memo/enumeration.rs deleted file mode 100644 index d849a5e75..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/enumeration.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! The candidate DAGs a memo stands for: each choice of one implementation -//! candidate per needed group, and the complete DAG that choice composes. -//! -//! A group is needed when a query's root is in it or a chosen candidate reads -//! it. A group read by several chosen candidates takes one candidate, so its -//! nodes are built once and shared; which groups are needed depends on the -//! candidates chosen above them. The count is therefore not a product over -//! groups. -//! -//! A choice is counted, not the DAG it composes: two candidates of a group -//! that differ only in what recomposing guarantees re-derives compose equal -//! DAGs. Composing checks only that a choice composes; input contracts, -//! accuracy at the roots, Stage 2 legality and Stage 3 cost stay with the -//! validator and the later stages. The memo's groups are acyclic, as every -//! memo step builds them, but candidates may read each other (a rollup of -//! a finer grouping whose candidate reads the coarser one): a choice taking -//! such candidates together composes no DAG and is not enumerated. - -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::{OperatorNode, QueryRoot, SchemaDerivationError}; - -use super::rules::reuse::replace; -use asap_accuracy::propagation::recompose_guarantees; -use asap_types::stages::Memo; -use asap_types::stages::MemoGroupId; - -/// One candidate (an index into `Memo::candidates[group]`) for each -/// group it names. [`build_logical_asap_dags_from_chosen_candidates`] gives a group it does not name its first -/// candidate. -pub type ChosenCandidatePerMemoGroup = BTreeMap; - -/// The choices [`enumerate`] found, in depth-first order. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct AllChosenCandidatesPerMemoGroup { - /// Each choice names exactly the groups it needs. - pub all_chosen_candidates: Vec, - /// False when the limit stopped the enumeration before every choice was found. - pub complete: bool, -} - -/// A choice that does not compose. -#[derive(Debug, thiserror::Error)] -pub enum ErrorBuildingLogicalASAPDagsFromChosenCandidates { - #[error("group {group:?} has no candidate {candidate}")] - NoSuchCandidate { - group: MemoGroupId, - candidate: usize, - }, - #[error("group {group:?}'s candidate {candidate} reads group {child:?} but has no node of it to replace")] - ChildNotInCandidate { - group: MemoGroupId, - candidate: usize, - child: MemoGroupId, - }, - /// The chosen candidates read each other in a cycle (e.g. a coarse - /// group's rollup reads a finer group whose chosen candidate reads it). - #[error("group {group:?} reads itself through the chosen candidates")] - Cycle { group: MemoGroupId }, - /// Re-deriving the composed nodes' schemas and guarantees failed. - #[error("recomposing the composed DAG: {0}")] - Recompose(#[from] SchemaDerivationError), -} - -/// Every choice of one candidate per needed group, starting from the query -/// roots, stopping after `limit` choices. A choice whose candidates read -/// each other in a cycle composes no DAG and is left out. -pub fn enumerate(memo: &Memo, limit: usize) -> AllChosenCandidatesPerMemoGroup { - fn walk( - memo: &Memo, - mut pending: Vec, - chosen_candidates: &mut ChosenCandidatePerMemoGroup, - limit: usize, - all_chosen_candidates: &mut Vec, - ) -> bool { - let group = loop { - match pending.pop() { - None => { - if all_chosen_candidates.len() == limit { - return false; - } - all_chosen_candidates.push(chosen_candidates.clone()); - return true; - } - Some(group) if !chosen_candidates.contains_key(&group) => break group, - Some(_) => {} - } - }; - for (index, candidate) in memo.candidates[group.0 as usize].iter().enumerate() { - // A candidate reading a group that already reads this one closes - // a cycle; every edge is checked as its group is chosen. - if candidate - .input_groups - .iter() - .any(|&child| reaches(memo, chosen_candidates, child, group)) - { - continue; - } - chosen_candidates.insert(group, index); - let mut next = pending.clone(); - next.extend(candidate.input_groups.iter().rev()); - let finished = walk(memo, next, chosen_candidates, limit, all_chosen_candidates); - chosen_candidates.remove(&group); - if !finished { - return false; - } - } - true - } - let roots = root_groups(memo).rev().collect(); - let mut all_chosen_candidates = Vec::new(); - let complete = walk( - memo, - roots, - &mut ChosenCandidatePerMemoGroup::new(), - limit, - &mut all_chosen_candidates, - ); - AllChosenCandidatesPerMemoGroup { - all_chosen_candidates, - complete, - } -} - -/// Each root group (one with a query's root) and its DAG under `chosen_candidates`: -/// each needed group implemented by its chosen candidate over its children's -/// composed DAGs, a group read several times composed once and shared. Every -/// node is then rebuilt with its derived schema and composed guarantee, as -/// in plan reconstruction. A scalar query's expression is not rebuilt: its -/// operator roots are root groups like any other query's. -pub fn build_logical_asap_dags_from_chosen_candidates( - memo: &Memo, - chosen_candidates: &ChosenCandidatePerMemoGroup, -) -> Result)>, ErrorBuildingLogicalASAPDagsFromChosenCandidates> -{ - fn composed( - memo: &Memo, - group: MemoGroupId, - chosen_candidates: &ChosenCandidatePerMemoGroup, - done: &mut HashMap>, - composing: &mut HashSet, - ) -> Result, ErrorBuildingLogicalASAPDagsFromChosenCandidates> { - if let Some(found) = done.get(&group) { - return Ok(Rc::clone(found)); - } - if !composing.insert(group) { - return Err(ErrorBuildingLogicalASAPDagsFromChosenCandidates::Cycle { group }); - } - let index = chosen_candidates.get(&group).copied().unwrap_or(0); - let candidate = memo.candidates[group.0 as usize].get(index).ok_or( - ErrorBuildingLogicalASAPDagsFromChosenCandidates::NoSuchCandidate { - group, - candidate: index, - }, - )?; - let own_nodes: HashSet<_> = OperatorNode::reachable(&candidate.root_node) - .iter() - .map(Rc::as_ptr) - .collect(); - let mut replacements = HashMap::new(); - for &child in &candidate.input_groups { - let old = &memo.groups[child.0 as usize].logical_node; - if !own_nodes.contains(&Rc::as_ptr(old)) { - return Err( - ErrorBuildingLogicalASAPDagsFromChosenCandidates::ChildNotInCandidate { - group, - candidate: index, - child, - }, - ); - } - replacements.insert( - Rc::as_ptr(old), - composed(memo, child, chosen_candidates, done, composing)?, - ); - } - let node = replace(&candidate.root_node, &replacements); - done.insert(group, Rc::clone(&node)); - Ok(node) - } - let mut done = HashMap::new(); - let mut composing = HashSet::new(); - let (groups, roots): (Vec<_>, Vec<_>) = root_groups(memo) - .map(|group| { - Ok(( - group, - QueryRoot::Operator(composed( - memo, - group, - chosen_candidates, - &mut done, - &mut composing, - )?), - )) - }) - .collect::, ErrorBuildingLogicalASAPDagsFromChosenCandidates>>()? - .into_iter() - .unzip(); - let roots = recompose_guarantees(&roots)?; - Ok(groups - .into_iter() - .zip(roots) - .map(|(group, root)| match root { - QueryRoot::Operator(node) => (group, node), - QueryRoot::Scalar(_) => unreachable!("composed from operator roots"), - }) - .collect()) -} - -/// Whether `from` is `to` or reads it through the candidates `chosen_candidates` names. -fn reaches( - memo: &Memo, - chosen_candidates: &ChosenCandidatePerMemoGroup, - from: MemoGroupId, - to: MemoGroupId, -) -> bool { - let mut pending = vec![from]; - let mut seen = HashSet::new(); - while let Some(group) = pending.pop() { - if group == to { - return true; - } - if !seen.insert(group) { - continue; - } - if let Some(&index) = chosen_candidates.get(&group) { - pending.extend(&memo.candidates[group.0 as usize][index].input_groups); - } - } - false -} - -fn root_groups(memo: &Memo) -> impl DoubleEndedIterator { - asap_types::stages::root_groups(&memo.query_roots).into_iter() -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules.rs b/crates/logical-asap-optimization/src/pass2/memo/rules.rs index e8eec23f0..c9b38ed85 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules.rs @@ -1,11 +1,19 @@ -//! Bounded logical memo rule expansion. Rules expose computations and typed -//! dependencies; physical configuration and cost selection live downstream. +//! Pass 2's rules over groups (concepts doc, section 3): each reads the +//! memo and adds groups (`Memo::find_or_add_group`) or candidates +//! (`Memo::add_candidate`); none copies a consumer. pub(crate) mod finer_aggregate; -pub(super) mod reuse; pub(crate) mod rollup; pub(crate) mod scan_labels; -pub(crate) mod shared_configuration_readers; -pub(super) mod shared_producer; pub(crate) mod summary_inputs; -pub(crate) mod summary_results; pub(crate) mod topk; + +/// Why a Pass 2 rule failed on a memo. +#[derive(Debug, thiserror::Error)] +pub enum Pass2RuleError { + #[error("memo contains an out-of-range group reference")] + InvalidGroupReference, + #[error("memo group capacity exceeds the group identifier range")] + TooManyGroups, + #[error("memo dependencies contain a cycle")] + CyclicDependencies, +} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs index 78aee9654..95c8c7c0a 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs @@ -1,177 +1,12 @@ //! Read a SQL aggregate from a finer aggregate's result //! ([`match_finer`](crate::pass2::finer_aggregate::match_finer)): filter the finer //! rows, then merge them into the query's groups with each measure's rollup. -use super::reuse::append_reader; -use crate::pass2::finer_aggregate::match_finer; -use crate::pass2::memo::state_groups::Pass2RuleError; -use asap_types::ir::operator::{AggIntent, Reduction}; -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, OperatorResultKind}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::HashMap; -use std::rc::Rc; -/// A memo group's SQL aggregate: no `without`, measure filter or `HAVING`. -struct Aggregate<'a> { - keys: &'a [usize], - measures: &'a [AggIntent], - output_names: &'a [String], - rows: &'a Rc, -} - -impl<'a> Aggregate<'a> { - fn of(node: &'a OperatorNode) -> Option { - if node.result_kind != OperatorResultKind::Relation { - return None; - } - let NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - measures, - output_names, - filters, - having: None, - child, - } = node.non_asap()? - else { - return None; - }; - // A same-level read takes the one row of each group, so a measure - // that gives several rows per group (a top-k, a count of values) - // does not qualify. - let one_row_per_group = !measures.iter().any(|measure| { - matches!( - measure, - AggIntent::TopK { .. } | AggIntent::CountValues { .. } - ) - }); - (!by.is_without() && filters.iter().all(Option::is_none) && one_row_per_group).then_some( - Self { - keys: by.keys(), - measures, - output_names, - rows: child, - }, - ) - } -} - -/// The measure that merges finer values of `measure` (at `col` of the finer -/// result) into a coarser group (Calcite's `SqlAggFunction.getRollup`); -/// `None` when its values do not roll up. A count rolls up as a sum only -/// into grouped queries: a global count of no rows is 0, a sum NULL. -fn value_rollup(measure: &AggIntent, col: usize, global: bool) -> Option { - match measure { - AggIntent::Sum { .. } => Some(AggIntent::Sum { col: Some(col) }), - AggIntent::Count { .. } if !global => Some(AggIntent::Sum { col: Some(col) }), - AggIntent::Min { .. } => Some(AggIntent::Min { col: Some(col) }), - AggIntent::Max { .. } => Some(AggIntent::Max { col: Some(col) }), - _ => None, - } -} - -pub(crate) fn add_finer_aggregate_candidates(memo: &mut Memo) -> Result { - // Only aggregates of equal measures can match; bucket by them (the - // first levels of the paper's filter tree) instead of trying all pairs. - let mut buckets: HashMap> = HashMap::new(); - for (group, entry) in memo.groups.iter().enumerate() { - if let Some(aggregate) = Aggregate::of(&entry.logical_node) { - buckets - .entry(format!("{:?}", aggregate.measures)) - .or_default() - .push(group); - } - } - let mut added = 0; - for groups in buckets.values() { - for &owner in groups { - for &finer in groups { - if owner != finer { - added += add_read(memo, owner, finer)?; - } - } - } - } - Ok(added) -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -/// Adds to group `owner` a read of group `finer`'s result when it computes -/// `owner`'s aggregate; the number of candidates added (0 or 1). -fn add_read(memo: &mut Memo, owner: usize, finer: usize) -> Result { - let query_node = Rc::clone(&memo.groups[owner].logical_node); - let finer_node = Rc::clone(&memo.groups[finer].logical_node); - let (Some(query), Some(fine)) = (Aggregate::of(&query_node), Aggregate::of(&finer_node)) else { - return Ok(0); - }; - let Some(matched) = match_finer(query.rows, query.keys, fine.rows, fine.keys) else { - return Ok(0); - }; - let global = query.keys.is_empty(); - // A finer measure's value column follows the finer keys. When each query - // group is at most one finer row, `MIN` reads that row's value, and over - // no row gives a global query SQL's one NULL row. - let measures = query - .measures - .iter() - .enumerate() - .map(|(index, measure)| { - let col = fine.keys.len() + index; - if !matched.needs_rollup && !(global && matches!(measure, AggIntent::Count { .. })) { - Some(AggIntent::Min { col: Some(col) }) - } else if matched.needs_rollup { - value_rollup(measure, col, global) - } else { - None - } - }) - .collect::>>(); - let Some(measures) = measures else { - return Ok(0); - }; - let reuse_error = - |error: asap_types::ir::SchemaDerivationError| Pass2RuleError::ReuseRule(error.to_string()); - let rows = match matched.compensating_filter { - Some(pred) => OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred, - child: Rc::clone(&finer_node), - })) - .map_err(reuse_error)?, - None => Rc::clone(&finer_node), - }; - let reader = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(matched.query_keys), - measures, - output_names: query.output_names.to_vec(), - filters: vec![], - having: None, - child: rows, - })) - .map_err(reuse_error)?; - let Some(original) = memo.candidates[owner] - .iter() - .find(|c| c.origin == OriginOfCandidateForMemoGroup::Pass1) - .cloned() - else { - return Ok(0); - }; - // The reader's columns take the query's qualifiers. A rollup whose type - // or nullability differs from the query's (a sum of counts) is not - // offered. - let mut schema = reader.schema.clone(); - for (field, expected) in schema - .fields - .iter_mut() - .zip(&original.root_node.schema.fields) - { - field.table = expected.table.clone(); - } - if schema != original.root_node.schema || reader.result_kind != original.root_node.result_kind { - return Ok(0); - } - let reader = Rc::new(OperatorNode::with_schema(reader.operator.clone(), schema)); - Ok(usize::from(append_reader( - memo, - owner, - &original, - reader, - OriginOfCandidateForMemoGroup::Rollup, - )?)) +/// Rule 5, finer aggregate: a candidate of a coarse group that filters and rolls up a finer group. +pub(crate) fn add_finer_aggregate_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 5 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/reuse.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/reuse.rs deleted file mode 100644 index 5eac3feb1..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/reuse.rs +++ /dev/null @@ -1,227 +0,0 @@ -//! Shared boundary handling for producer-level reuse rules. Rule outputs retain -//! canonical input identities after guarantee recomposition rebuilds their DAGs. -use crate::pass2::memo::state_groups::{boundary_groups, Pass2RuleError}; -use asap_types::ir::OperatorNode; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId}; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -pub(in crate::pass2::memo) fn eligible(candidate: &CandidateForMemoGroup) -> bool { - candidate.accuracy_budget_parts > 0 -} - -pub(in crate::pass2::memo) fn restore_inputs( - node: &Rc, - memo: &Memo, - inputs: &[MemoGroupId], -) -> Rc { - // These are the same declared logical input groups, rebound after a rule - // cloned the DAG. Value guarantees are recomposed from selected producers - // during reconstruction; state inputs additionally require the same error - // contract because the intermediate state's guarantee is part of its API. - let canonical: Vec<_> = inputs - .iter() - .flat_map(|id| OperatorNode::reachable(&memo.groups[id.0 as usize].logical_node)) - .collect(); - fn visit( - node: &Rc, - canonical: &[Rc], - done: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if let Some(found) = done.get(&Rc::as_ptr(node)) { - return Rc::clone(found); - } - let operator = node - .operator - .map_children(|child| visit(child, canonical, done)); - let rebuilt = Rc::new(OperatorNode { - operator, - ..(**node).clone() - }); - let result = canonical - .iter() - .find(|known| { - same_node( - known, - &rebuilt, - known.result_kind == asap_types::ir::OperatorResultKind::State, - ) - }) - .cloned() - .unwrap_or(rebuilt); - done.insert(Rc::as_ptr(node), Rc::clone(&result)); - result - } - visit(node, &canonical, &mut HashMap::new()) -} - -pub(in crate::pass2::memo) fn same_node( - left: &OperatorNode, - right: &OperatorNode, - guarantees: bool, -) -> bool { - left.operator.map_children(|_| ()) == right.operator.map_children(|_| ()) - && left.schema == right.schema - && left.result_kind == right.result_kind - && left.timing == right.timing - && left.materialization == right.materialization - && left.children().len() == right.children().len() - && left - .children() - .iter() - .zip(right.children()) - .all(|(a, b)| Rc::ptr_eq(a, b)) - && serde_json::to_string(&( - left.operator.map_children(|_| ()), - &left.schema, - left.result_kind, - left.timing, - left.materialization, - )) - .ok() - == serde_json::to_string(&( - right.operator.map_children(|_| ()), - &right.schema, - right.result_kind, - right.timing, - right.materialization, - )) - .ok() - && (!guarantees || left.guarantee == right.guarantee) -} - -pub(in crate::pass2::memo) fn append_producer( - memo: &mut Memo, - node: Rc, - queries: &[usize], - origin: OriginOfCandidateForMemoGroup, - accuracy_budget_parts: usize, -) -> Result { - // Reuse only dedicated producer groups. A logical result group can contain - // other implementations and cannot enforce this producer's exact contract. - let root_groups = asap_types::stages::root_groups(&memo.query_roots); - if let Some((index, _)) = memo.groups.iter().enumerate().find(|(index, group)| { - !root_groups.contains(&MemoGroupId(*index as u32)) - && memo.candidates[*index].len() == 1 - && memo.candidates[*index][0].origin == origin - && memo.candidates[*index][0].accuracy_budget_parts == accuracy_budget_parts - && same_node(&group.logical_node, &node, true) - }) { - return Ok(MemoGroupId(index as u32)); - } - let id = - MemoGroupId(u32::try_from(memo.groups.len()).map_err(|_| Pass2RuleError::TooManyGroups)?); - let node = Rc::new((*node).clone()); - let boundaries = memo - .groups - .iter() - .enumerate() - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - let children = boundary_groups(&node, &boundaries); - memo.groups.push(MemoGroup { - logical_node: Rc::clone(&node), - child_groups: children.clone(), - parents: Vec::new(), - queries: queries.to_vec(), - }); - memo.candidates.push(vec![CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: node, - input_groups: children, - origin, - accuracy_budget_parts, - }]); - Ok(id) -} - -pub(in crate::pass2::memo) fn append_reader( - memo: &mut Memo, - owner: usize, - original: &CandidateForMemoGroup, - node: Rc, - origin: OriginOfCandidateForMemoGroup, -) -> Result { - if original.root_node.schema.fields != node.schema.fields - || original.root_node.schema.time_index != node.schema.time_index - || original.root_node.schema.unique_keys != node.schema.unique_keys - || (original.root_node.schema.closed && !node.schema.closed) - || original.root_node.result_kind != node.result_kind - { - return Err(Pass2RuleError::ReuseRule(format!( - "reuse changed the reader output contract ({origin:?}): expected {:?}, got {:?}", - original.root_node.schema, node.schema - ))); - } - let boundaries = memo - .groups - .iter() - .enumerate() - .filter(|(id, _)| *id != owner) - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - let children = boundary_groups(&node, &boundaries); - if memo.candidates[owner].iter().any(|known| { - known.input_groups == children - && known.accuracy_budget_parts == original.accuracy_budget_parts - && *known.root_node == *node - && serde_json::to_string(&known.root_node).ok() == serde_json::to_string(&node).ok() - }) { - return Ok(false); - } - let mut candidate = original.clone(); - candidate.root_node = node; - candidate.input_groups = children; - candidate.origin = origin; - memo.candidates[owner].push(candidate); - Ok(true) -} - -pub(in crate::pass2::memo) fn replace( - node: &Rc, - replacements: &HashMap<*const OperatorNode, Rc>, -) -> Rc { - fn visit( - node: &Rc, - replacements: &HashMap<*const OperatorNode, Rc>, - done: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if let Some(found) = replacements - .get(&Rc::as_ptr(node)) - .or_else(|| done.get(&Rc::as_ptr(node))) - { - return Rc::clone(found); - } - let mut changed = false; - let operator = node.operator.map_children(|child| { - let replacement = visit(child, replacements, done); - changed |= !Rc::ptr_eq(child, &replacement); - replacement - }); - let rebuilt = if changed { - Rc::new(OperatorNode { - operator, - ..(**node).clone() - }) - } else { - Rc::clone(node) - }; - done.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); - rebuilt - } - visit(node, replacements, &mut HashMap::new()) -} - -pub(in crate::pass2::memo) fn combined_inputs( - left: &CandidateForMemoGroup, - right: &CandidateForMemoGroup, -) -> Vec { - let mut seen = HashSet::new(); - left.input_groups - .iter() - .chain(&right.input_groups) - .copied() - .filter(|id| seen.insert(*id)) - .collect() -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs index 1f684e04f..de56a825b 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs @@ -1,360 +1,18 @@ //! Producer-level exact and sketch rollups, using the existing capability rules. -use super::reuse::{ - append_producer, append_reader, combined_inputs, eligible, replace, restore_inputs, -}; -use crate::pass2::asap_aware_cse::{rollup_candidate_for, rollup_pass2_candidates}; -use crate::pass2::memo::state_groups::{simple_build, Pass2RuleError}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ - ASAPOp, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, -}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::HashMap; -use std::rc::Rc; -pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result { - let mut candidates = Vec::new(); - for (group, entries) in memo.candidates.iter().enumerate() { - for candidate in entries { - if eligible(candidate) - && matches!( - candidate.origin, - OriginOfCandidateForMemoGroup::Pass1 - | OriginOfCandidateForMemoGroup::WidenedScans - ) - { - if let Some((build, _)) = simple_build(memo, candidate) { - candidates.push((group, candidate.clone(), build)); - } - } - } - } - let mut added = 0; - for (coarse_group, coarse, coarse_build) in &candidates { - for (fine_group, fine, _) in &candidates { - if coarse_group == fine_group { - continue; - } - let roots = [ - QueryRoot::Operator(Rc::clone(&coarse.root_node)), - QueryRoot::Operator(Rc::clone(&fine.root_node)), - ]; - let Some(rule) = rollup_candidate_for(&roots, Rc::as_ptr(coarse_build)) else { - continue; - }; - if !OperatorNode::reachable(&fine.root_node) - .iter() - .any(|node| Rc::ptr_eq(node, &rule.finer)) - { - continue; - } - let state_rollup = OperatorNode::reachable(&rule.rollup).iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - ) - }); - let is_sketch = matches!( - rule.finer.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ); - let rewritten = rollup_pass2_candidates(&roots, &[rule]) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let Some(result) = rewritten.into_iter().next() else { - continue; - }; - // Existing rule API marks individually noncomposable rewrites as - // invalid candidates; retain the local implementation in that case. - let Ok(rewritten) = result.roots else { - continue; - }; - let [QueryRoot::Operator(coarse_node), QueryRoot::Operator(fine_node)] = - rewritten.as_slice() - else { - unreachable!("two operator roots") - }; - let inputs = combined_inputs(coarse, fine); - let coarse_node = restore_inputs(coarse_node, memo, &inputs); - let fine_node = restore_inputs(fine_node, memo, &inputs); - if is_sketch || state_rollup { - // Concrete resized states become ordinary state groups after all - // producer-level configurations have been expanded. - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - added += usize::from(append_reader( - memo, - *fine_group, - fine, - fine_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - } else { - let Some(value) = - OperatorNode::reachable(&coarse_node) - .into_iter() - .find_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { child, .. }) - if matches!( - child.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - ) => - { - Some(Rc::clone(child)) - } - _ => None, - }) - else { - continue; - }; - // When the finer query's result is the value itself and every - // implementation of it is exact, read the finer query's group: - // one stored result then serves both queries, as the - // whole-workload rewrite does, without a separate producer the - // finer query would read through an identity projection. - // Exactness is checked when this rule runs; a later - // implementation of the group is still checked by plan - // reconstruction (input contract) and Stage 3 (guarantee). - let fine_result_is_value = *fine_node == *value - && memo.groups[*fine_group].logical_node.schema == value.schema - && memo.candidates[*fine_group].iter().all(|candidate| { - candidate - .root_node - .guarantee - .as_ref() - .is_some_and(|guarantee| guarantee.is_exact()) - }); - if fine_result_is_value { - let fine_result = Rc::clone(&memo.groups[*fine_group].logical_node); - let coarse_node = replace( - &coarse_node, - &HashMap::from([(Rc::as_ptr(&value), fine_result)]), - ); - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - continue; - } - let mut queries = memo.groups[*coarse_group].queries.clone(); - queries.extend(&memo.groups[*fine_group].queries); - queries.sort_unstable(); - queries.dedup(); - let producer = append_producer( - memo, - Rc::clone(&value), - &queries, - OriginOfCandidateForMemoGroup::Rollup, - 1, - )?; - let canonical = Rc::clone(&memo.groups[producer.0 as usize].logical_node); - let coarse_node = replace( - &coarse_node, - &HashMap::from([(Rc::as_ptr(&value), Rc::clone(&canonical))]), - ); - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - if let Some(own_value) = OperatorNode::reachable(&fine_node) - .into_iter() - .find(|node| **node == *value) - { - let fine_reader = if Rc::ptr_eq(&own_value, &fine_node) { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: canonical - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: Rc::clone(&canonical), - })) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))? - } else { - replace( - &fine_node, - &HashMap::from([(Rc::as_ptr(&own_value), Rc::clone(&canonical))]), - ) - }; - added += usize::from(append_reader( - memo, - *fine_group, - fine, - fine_reader, - OriginOfCandidateForMemoGroup::Rollup, - )?); - } - } - } - } - Ok(added) -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -/// Existing tumbling window rollup rules applied to producer pairs on already generated -/// legal local grids. No whole-workload choice combinations are constructed. -pub(crate) fn add_window_rollup_candidates(memo: &mut Memo) -> Result { - use crate::pass2::asap_aware_cse::{ - tumbling_window_candidates, window_composition_shared_chain_candidates, - }; - let mut grids = vec![std::collections::BTreeSet::new(); memo.groups.len()]; - for (group, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates.iter().filter(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::TumblingWindows - && eligible(candidate) - }) { - if let Some((build, _)) = simple_build(memo, candidate) { - if let Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) = build.asap() - { - grids[group].insert(*tumbling_window_width_ms); - } - } - } - } - let candidates: Vec<_> = memo - .candidates - .iter() - .enumerate() - .flat_map(|(group, candidates)| { - candidates - .iter() - .filter(|candidate| { - eligible(candidate) && candidate.origin == OriginOfCandidateForMemoGroup::Pass1 - }) - .map(move |candidate| (group, candidate.clone())) - }) - .collect(); - let mut added = 0; - for (coarse_group, coarse) in &candidates { - for (fine_group, fine) in &candidates { - if coarse_group == fine_group || grids[*fine_group].is_empty() { - continue; - } - for grid in &grids[*coarse_group] { - let roots = [ - QueryRoot::Operator(Rc::clone(&coarse.root_node)), - QueryRoot::Operator(Rc::clone(&fine.root_node)), - ]; - let tumbling_windows = - tumbling_window_candidates(&roots, &[Some(*grid), Some(*grid)]); - if tumbling_windows.len() != 2 { - continue; - } - for rewritten in - window_composition_shared_chain_candidates(&roots, &tumbling_windows) - { - if rewritten.shared_chains.is_empty() { - continue; - } - let Ok(nodes) = rewritten.roots else { continue }; - let [QueryRoot::Operator(coarse_node), QueryRoot::Operator(fine_node)] = - nodes.as_slice() - else { - continue; - }; - if !OperatorNode::reachable(coarse_node).iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - ) - }) { - continue; - } - let inputs = combined_inputs(coarse, fine); - let coarse_node = restore_inputs(coarse_node, memo, &inputs); - let fine_node = restore_inputs(fine_node, memo, &inputs); - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - added += usize::from(append_reader( - memo, - *fine_group, - fine, - fine_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - } - } - } - } - Ok(added) +/// Rule 6, rollup: a candidate of a coarse aggregate or summary group reading +/// a finer group whose state rolls up (exact or sketch merge). +pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 6 over groups") } -#[cfg(test)] -mod tests { - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{lower_promql, sharing_rules_and_shared_state_groups}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::QueryRoot; - use asap_types::stages::OriginOfCandidateForMemoGroup; - - // When the finer query's own result is the finalized value the coarse - // query rolls up, the rollup reads the finer query's group, so one - // stored result serves both queries without a copy or an identity - // projection. - #[test] - fn a_rollup_reads_the_finer_query_result_itself() { - let roots: Vec<_> = ["sum by (job) (m)", "sum by (job, instance) (m)"] - .into_iter() - .map(|text| QueryRoot::Operator(lower_promql(text, AccuracyTarget::exact()))) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let memo = - sharing_rules_and_shared_state_groups(try_populate_pass1(initial, &inventory).unwrap()) - .unwrap(); - let root_of = |query: usize| memo.query_roots[query].as_operator().unwrap().0 as usize; - let (coarse, fine) = (root_of(0), root_of(1)); - assert!( - memo.candidates[coarse].iter().any(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::Rollup - && candidate - .input_groups - .iter() - .any(|child| child.0 as usize == fine) - }), - "the coarse rollup reads the finer query's group" - ); - assert!( - !memo.candidates[fine] - .iter() - .any(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup), - "the finer query needs no reader of a separate producer" - ); - } +/// Rule 7, window rollup: a candidate of a longer-range group merging a +/// shorter-range group's windows. +pub(crate) fn add_window_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 7 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs index 1410c426a..a808eb48f 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs @@ -1,237 +1,11 @@ //! Scan-label widening for explicit memo state sharing. Reuses the existing //! Pass 2 widening rule; only its source-chain boundaries are interned here. -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; -use asap_accuracy::propagation::recompose_guarantees; -use asap_types::ir::operator::Reduction; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot}; +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -use crate::pass2::memo::state_groups::{boundary_groups, simple_build, Pass2RuleError}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId}; - -/// Preserve local implementations and append readers over union-label inputs. -/// Parent/query reachability is rebuilt by the enclosing state-group expansion. -pub(crate) fn add_scan_label_candidates(memo: &mut Memo) -> Result { - let mut owners = Vec::new(); - let mut roots = Vec::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - // Source-group candidates expose scan reuse to every downstream - // operator, including exact/non-summary computations. Their public - // schemas stay narrow through the existing projection rewrite. - if is_source_chain(&candidate.root_node) && super::reuse::eligible(candidate) { - owners.push((group, index)); - roots.push(QueryRoot::Operator(Rc::clone(&candidate.root_node))); - continue; - } - let Some((build, children)) = simple_build(memo, candidate) else { - continue; - }; - if !matches!( - build.asap(), - Some( - ASAPOp::SummaryAgg { - reduction: Reduction::PerEntity, - filter: None, - .. - } | ASAPOp::SummaryTumblingWindowChain { - reduction: Reduction::PerEntity, - filter: None, - .. - } - ) - ) || candidate - .input_groups - .iter() - .any(|child| !children.contains(child)) - || children - .iter() - .any(|child| !is_source_chain(&memo.groups[child.0 as usize].logical_node)) - { - continue; - } - owners.push((group, index)); - roots.push(QueryRoot::Operator(Rc::clone(&candidate.root_node))); - } - } - let Some(widened) = crate::pass2::asap_aware_cse::with_union_scan_labels(&roots) - .map_err(|error| Pass2RuleError::ScanLabelWidening(error.to_string()))? - else { - return Ok(0); - }; - // Widening recomposes every root's guarantees, also those it left - // unchanged, so a root is unchanged when it equals its recomposed - // original. Compared with the stored original instead, an unchanged root - // (a scan that already has every label) that differs only in its - // guarantee annotation would pass for a widened reader, and interning - // would map it back to its own group: a candidate reading the group it - // implements. - let unwidened = recompose_guarantees(&roots) - .map_err(|error| Pass2RuleError::ScanLabelWidening(error.to_string()))?; - let mut added = 0; - let mut canonical = HashMap::new(); - for (((group, index), root), unwidened) in owners.into_iter().zip(widened).zip(unwidened) { - let (QueryRoot::Operator(node), QueryRoot::Operator(unwidened)) = (root, unwidened) else { - unreachable!("operator readers") - }; - let original = memo.candidates[group][index].clone(); - if *node == *unwidened || !same_output_contract(&original.root_node, &node) { - continue; - } - let queries = memo.groups[group].queries.clone(); - let mut node = intern_inputs(&node, memo, &queries, &mut canonical)?; - if memo - .groups - .iter() - .any(|group| Rc::ptr_eq(&group.logical_node, &node)) - { - // Interning an equivalent range/source can return another group's - // canonical value. Keep a concrete reader instead of a root alias. - node = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - cols: node - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| asap_types::ir::ProjectItem { - alias: Some(field.name.clone()), - expr: asap_types::ir::ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: node, - })) - .map_err(|error| Pass2RuleError::ScanLabelWidening(error.to_string()))?; - if !same_output_contract(&original.root_node, &node) { - continue; - } - } - let boundaries = memo - .groups - .iter() - .enumerate() - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - let mut reader = original; - reader.root_node = node; - reader.input_groups = boundary_groups(&reader.root_node, &boundaries); - reader.origin = OriginOfCandidateForMemoGroup::WidenedScans; - if memo.candidates[group].iter().any(|known| { - known.origin == OriginOfCandidateForMemoGroup::WidenedScans - && known.input_groups == reader.input_groups - && *known.root_node == *reader.root_node - }) { - continue; - } - memo.candidates[group].push(reader); - added += 1; - } - Ok(added) -} - -fn same_output_contract(expected: &OperatorNode, actual: &OperatorNode) -> bool { - expected.result_kind == actual.result_kind - && expected.schema.fields == actual.schema.fields - && expected.schema.time_index == actual.schema.time_index - && expected.schema.unique_keys == actual.schema.unique_keys - // A projection may enumerate every output of an originally open schema. - && (!expected.schema.closed || actual.schema.closed) -} - -fn is_source_chain(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { .. }) => true, - Some(NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }) => { - is_source_chain(child) - } - _ => false, - } -} - -fn same_source_node(left: &OperatorNode, right: &OperatorNode) -> bool { - // These operators preserve exact source rows; recomposition's exact - // guarantee annotation does not change their logical/execution contract. - left.operator.map_children(|_| ()) == right.operator.map_children(|_| ()) - && left.schema == right.schema - && left.result_kind == right.result_kind - && left.timing == right.timing - && left.materialization == right.materialization - && left - .children() - .iter() - .zip(right.children()) - .all(|(a, b)| Rc::ptr_eq(a, b)) - && left.children().len() == right.children().len() -} - -fn intern_inputs( - node: &Rc, - memo: &mut Memo, - queries: &[usize], - done: &mut HashMap<*const OperatorNode, Rc>, -) -> Result, Pass2RuleError> { - if let Some(known) = done.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(known)); - } - let mut children = HashMap::new(); - for child in node.children() { - children.insert( - Rc::as_ptr(child), - intern_inputs(child, memo, queries, done)?, - ); - } - let rewritten = Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| Rc::clone(&children[&Rc::as_ptr(child)])), - ..(**node).clone() - }); - let result = if is_source_chain(&rewritten) { - if let Some(existing) = memo - .groups - .iter() - .find(|group| same_source_node(&group.logical_node, &rewritten)) - { - Rc::clone(&existing.logical_node) - } else { - // The new group's id must fit a `MemoGroupId`. - u32::try_from(memo.groups.len()).map_err(|_| Pass2RuleError::TooManyGroups)?; - let mut seen = HashSet::new(); - let inputs: Vec<_> = rewritten - .children() - .iter() - .map(|child| { - MemoGroupId( - memo.groups - .iter() - .position(|group| Rc::ptr_eq(child, &group.logical_node)) - .expect("source-chain children were interned") - as u32, - ) - }) - .filter(|input| seen.insert(*input)) - .collect(); - memo.groups.push(MemoGroup { - logical_node: Rc::clone(&rewritten), - child_groups: inputs.clone(), - parents: Vec::new(), - queries: queries.to_vec(), - }); - memo.candidates.push(vec![CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: Rc::clone(&rewritten), - input_groups: inputs, - origin: OriginOfCandidateForMemoGroup::WidenedScans, - accuracy_budget_parts: 1, - }]); - rewritten - } - } else { - rewritten - }; - done.insert(Rc::as_ptr(node), Rc::clone(&result)); - Ok(result) +/// Rule 2, scan widening: a group for the widened scan, and in each original scan group a candidate `Project(widened)`. +pub(crate) fn add_scan_label_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 2 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs deleted file mode 100644 index 5d45eac0d..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs +++ /dev/null @@ -1,637 +0,0 @@ -//! Expand compatible readers at finite shared sizes and tumbling-window horizons before -//! interning concrete states. Local candidates remain available. -use crate::stage0::cse::same_value; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::flat::{flatten, FlatDag}; -use asap_types::ir::schema::{sketch::SketchKind, FieldDataType}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; - -use super::shared_producer::{ - add_shared_producer_candidates, spans_groups, Reader, SharedProducer, -}; -use crate::pass2::asap_aware_cse::{with_resized_build, with_tumbling_window_chain}; -use crate::pass2::memo::shared_configurations::{ - shared_size, state_size_configurations, tumbling_window_configurations, - TumblingWindowConfiguration, TumblingWindowRequirement, -}; -use crate::pass2::memo::state_groups::{ - boundary_ptrs, same_range_input, simple_build, Pass2RuleError, -}; -use asap_types::stages::Memo; - -/// `resized` (a rewrite of `original` with the same shape) with each node -/// that is a child group's node in `original` put back, and the nodes above -/// rebuilt over them, keeping their own schema and guarantee. -fn with_child_nodes( - original: &Rc, - resized: &Rc, - child_nodes: &HashSet<*const OperatorNode>, - done: &mut HashMap<*const OperatorNode, Rc>, -) -> Rc { - if child_nodes.contains(&Rc::as_ptr(original)) { - return Rc::clone(original); - } - if let Some(found) = done.get(&Rc::as_ptr(resized)) { - return Rc::clone(found); - } - let pairs: HashMap<*const OperatorNode, Rc> = original - .children() - .into_iter() - .zip(resized.children()) - .map(|(o, r)| (Rc::as_ptr(r), with_child_nodes(o, r, child_nodes, done))) - .collect(); - let mut changed = false; - let operator = resized.operator.map_children(|child| { - let new = Rc::clone(&pairs[&Rc::as_ptr(child)]); - changed |= !Rc::ptr_eq(&new, child); - new - }); - let out = match changed { - false => Rc::clone(resized), - true => Rc::new( - OperatorNode::with_schema(operator, resized.schema.clone()) - .with_guarantee(resized.guarantee.clone()) - .with_timing(resized.timing) - .with_materialization(resized.materialization), - ), - }; - done.insert(Rc::as_ptr(resized), Rc::clone(&out)); - out -} - -/// A summary build one candidate reads: its own need of the shared producer. -struct StateNeed { - candidate: usize, - build: Rc, - merge: Option>, - family: FieldDataType, - /// The reader candidate's `accuracy_budget_parts`: the shares its build - /// is sized for. - accuracy_budget_parts: usize, - /// The reader's normalized tumbling-window layout, when its set shares tumbling windows: equal - /// layouts of one set have one index, so a configuration rewrites them once. - normalized_layout: Option, -} - -/// A shared build's need: one tumbling-window layout (its index among the set's tumbling window -/// configurations, for request deduplication) or one size, with the shares -/// of the budget its build is sized for ([`covered_parts`]). -enum StateCovering { - TumblingWindows { - configuration_index: usize, - configuration: TumblingWindowConfiguration, - accuracy_budget_parts: usize, - }, - Size { - family: FieldDataType, - accuracy_budget_parts: usize, - }, -} - -/// Summary builds of one family over one input at different sizes or tumbling window -/// layouts: each reader is rewritten to a build at each covering size or -/// layout that serves it, and equal concrete builds become one state group -/// later ([`super::super::state_groups`]). -struct SummaryStates { - deduplicate_requests: bool, - /// Every candidate's graph signature by group, so no rewrite repeats one. - known: HashMap<(usize, String), Vec>>, - rewrite_count: usize, - /// The (normalized layout, configuration index) rewrites already made. - requests: HashSet<(usize, usize)>, -} - -pub(crate) fn add_state_configuration_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - expand_configurations(memo, true).map(|_| ()) -} - -fn expand_configurations( - memo: &mut Memo, - deduplicate_requests: bool, -) -> Result { - let mut states = SummaryStates { - deduplicate_requests, - known: HashMap::new(), - rewrite_count: 0, - requests: HashSet::new(), - }; - add_shared_producer_candidates(memo, &mut states)?; - Ok(states.rewrite_count) -} - -impl SharedProducer for SummaryStates { - type Need = StateNeed; - type Covering = StateCovering; - - fn sets(&mut self, memo: &Memo) -> Result>>, Pass2RuleError> { - let mut sets: Vec>> = Vec::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (candidate, entry) in candidates.iter().enumerate() { - let Some((build, _)) = simple_build(memo, entry) else { - continue; - }; - // Configuration expansion precedes Stage 2. The rewrite helpers - // create unassigned state; preserve any already assigned build. - if build.timing.is_some() - || build.materialization - != asap_types::physical::Materialization::NotMaterialized - { - continue; - } - let family = match build.asap() { - Some( - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => family.clone(), - _ => unreachable!(), - }; - let merge = if matches!( - build.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) { - let merges: Vec<_> = OperatorNode::reachable(&entry.root_node).into_iter().filter(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryMerge {children, group_by: None, ..}) - if children.len() == 1 && Rc::ptr_eq(&children[0], &build)) - }).collect(); - // The existing rewrite changes one plain merge. Other tumbling window uses - // need their own explicit transformation, not guessed semantics. - if merges.len() != 1 { - continue; - } - Some(Rc::clone(&merges[0])) - } else { - None - }; - let reader = Reader { - group, - need: StateNeed { - candidate, - build, - merge, - family, - accuracy_budget_parts: entry.accuracy_budget_parts, - normalized_layout: None, - }, - }; - match sets - .iter_mut() - .find(|set| compatible(&set[0].need, &reader.need)) - { - Some(set) => set.push(reader), - None => sets.push(vec![reader]), - } - } - } - for (group, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates { - self.known - .entry((group, graph_signature(&candidate.root_node)?)) - .or_default() - .push(Rc::clone(&candidate.root_node)); - } - } - let mut layouts: Vec<(usize, usize, FlatDag)> = Vec::new(); - for set in &mut sets { - if spans_groups(set) { - assign_normalized_layouts(memo, set, &mut layouts)?; - } - } - Ok(sets) - } - - fn coverings(&self, set: &[Reader]) -> Result, Pass2RuleError> { - let tumbling_window_requirements: Option> = set - .iter() - .map(|reader| tumbling_window_requirement(&reader.need)) - .collect(); - let Some(requirements) = tumbling_window_requirements else { - let requirements = set - .iter() - .map(|reader| match &reader.need.family { - FieldDataType::Sketch(kind, _) => Some(kind.params().clone()), - _ => None, - }) - .collect(); - let first = &set[0].need.family; - return Ok(state_size_configurations(requirements) - .into_iter() - .map(|configuration| { - let family = match (first, configuration) { - (FieldDataType::Sketch(kind, grouping), Some(params)) => { - FieldDataType::Sketch( - SketchKind::new(*kind.algorithm(), params), - grouping.clone(), - ) - } - _ => first.clone(), - }; - StateCovering::Size { - accuracy_budget_parts: covered_parts(&family, set), - family, - } - }) - .collect()); - }; - Ok(tumbling_window_configurations(&requirements) - .into_iter() - .enumerate() - .map( - |(configuration_index, configuration)| StateCovering::TumblingWindows { - configuration_index, - accuracy_budget_parts: covered_parts(&configuration.family, set), - configuration, - }, - ) - .collect()) - } - - fn derive( - &mut self, - memo: &mut Memo, - reader: &Reader, - covering: &StateCovering, - ) -> Result { - let need = &reader.need; - let serves = match covering { - StateCovering::TumblingWindows { configuration, .. } => configuration - .serves(&tumbling_window_requirement(need).expect("tumbling window set")), - StateCovering::Size { family, .. } => serves_size(family, &need.family), - }; - if !serves { - return Ok(0); - } - let original = memo.candidates[reader.group][need.candidate].clone(); - let roots = [QueryRoot::Operator(Rc::clone(&original.root_node))]; - let result = match covering { - StateCovering::TumblingWindows { - configuration_index, - configuration, - .. - } => { - let Ok(count) = u32::try_from( - configuration.longest_window_ms / configuration.tumbling_window_width_ms, - ) else { - return Ok(0); - }; - if self.deduplicate_requests - && need - .normalized_layout - .is_some_and(|layout| !self.requests.insert((layout, *configuration_index))) - { - return Ok(0); - } - self.rewrite_count += 1; - with_tumbling_window_chain( - &roots, - need.merge.as_ref().unwrap(), - configuration.family.clone(), - configuration.tumbling_window_width_ms, - count, - ) - } - StateCovering::Size { family, .. } => { - self.rewrite_count += 1; - with_resized_build(&roots, &need.build, family.clone()) - } - }; - let boundaries = boundary_ptrs(memo, &original); - let roots = - result.map_err(|error| Pass2RuleError::StateConfiguration(error.to_string()))?; - let [QueryRoot::Operator(node)] = roots.as_slice() else { - unreachable!("operator rewrite") - }; - let node = with_child_nodes(&original.root_node, node, &boundaries, &mut HashMap::new()); - // Resizing affects internal state types, never public output fields. - if original.root_node.schema.fields != node.schema.fields - || original.root_node.schema.time_index != node.schema.time_index - || original.root_node.schema.unique_keys != node.schema.unique_keys - || (original.root_node.schema.closed && !node.schema.closed) - { - return Err(Pass2RuleError::StateConfiguration( - "state rewrite changed the reader output contract".into(), - )); - } - let previous = self - .known - .entry((reader.group, graph_signature(&node)?)) - .or_default(); - if previous.iter().any(|known| **known == *node) { - return Ok(0); - } - previous.push(Rc::clone(&node)); - let mut candidate = original; - candidate.root_node = node; - // The rewritten build is sized for the covering's shares, at least - // the reader's own. - candidate.accuracy_budget_parts = match covering { - StateCovering::TumblingWindows { - accuracy_budget_parts, - .. - } - | StateCovering::Size { - accuracy_budget_parts, - .. - } => (*accuracy_budget_parts).max(candidate.accuracy_budget_parts), - }; - memo.candidates[reader.group].push(candidate); - Ok(1) - } -} - -/// Numbers the normalized tumbling-window layouts of `set`'s readers, when it shares -/// tumbling windows: different original layouts can produce the same requested -/// reader/configuration, so each is normalized once and its rewrite made once, -/// instead of once for every existing tumbling-window width. Numbers continue across -/// sets (`layouts`), so a configuration index of one set never matches -/// another's. -fn assign_normalized_layouts( - memo: &Memo, - set: &mut [Reader], - layouts: &mut Vec<(usize, usize, FlatDag)>, -) -> Result<(), Pass2RuleError> { - let Some(requirements) = set - .iter() - .map(|reader| tumbling_window_requirement(&reader.need)) - .collect::>>() - else { - return Ok(()); - }; - let mut layout_keys: HashMap<(usize, String), Vec> = HashMap::new(); - for (reader, requirement) in set.iter_mut().zip(&requirements) { - let Ok(count) = u32::try_from(requirement.window_ms) else { - // Normalization must not narrow the legal long-window domain. - continue; - }; - let need = &reader.need; - let original = &memo.candidates[reader.group][need.candidate]; - let roots = [QueryRoot::Operator(Rc::clone(&original.root_node))]; - let normalized = with_tumbling_window_chain( - &roots, - need.merge.as_ref().unwrap(), - need.family.clone(), - 1, - count, - ) - .map_err(|error| Pass2RuleError::StateConfiguration(error.to_string()))?; - let flat = flatten(&normalized).0; - let signature = serde_json::to_string(&flat) - .map_err(|error| Pass2RuleError::StateConfiguration(error.to_string()))?; - let indices = layout_keys.entry((reader.group, signature)).or_default(); - let existing = indices.iter().copied().find(|&index| { - let (group, candidate, previous) = &layouts[index]; - let previous_candidate = &memo.candidates[*group][*candidate]; - previous == &flat - && previous_candidate.origin == original.origin - && previous_candidate.input_groups == original.input_groups - && previous_candidate.fused_groups == original.fused_groups - && previous_candidate.accuracy_budget_parts == original.accuracy_budget_parts - }); - reader.need.normalized_layout = Some(existing.unwrap_or_else(|| { - let index = layouts.len(); - layouts.push((reader.group, need.candidate, flat)); - indices.push(index); - index - })); - } - Ok(()) -} - -fn graph_signature(node: &Rc) -> Result { - serde_json::to_string(&flatten(&[QueryRoot::Operator(Rc::clone(node))]).0) - .map_err(|error| Pass2RuleError::StateConfiguration(error.to_string())) -} - -fn tumbling_window_requirement(reader: &StateNeed) -> Option { - let ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - } = reader.build.asap()? - else { - return None; - }; - let ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - } = reader.merge.as_ref()?.asap()? - else { - return None; - }; - Some(TumblingWindowRequirement { - tumbling_window_width_ms: *tumbling_window_width_ms, - window_ms: tumbling_window_width_ms.checked_mul(u64::from( - newest_tumbling_windows.unwrap_or(*tumbling_window_count), - ))?, - family: reader.family.clone(), - }) -} - -/// Whether a build of `family` serves a reader of `reader`: the size both -/// share is `family` itself. -fn serves_size(family: &FieldDataType, reader: &FieldDataType) -> bool { - shared_size(family, reader).is_some_and(|joined| same_value(&joined, family)) -} - -/// The shares of the budget a build of `family` is sized for: the most of -/// any reader in `set` it serves (it is at least as large as that reader's -/// own build, sized for those shares); 1 when it serves none. -fn covered_parts(family: &FieldDataType, set: &[Reader]) -> usize { - set.iter() - .filter(|reader| serves_size(family, &reader.need.family)) - .map(|reader| reader.need.accuracy_budget_parts) - .max() - .unwrap_or(1) -} - -fn compatible(left: &StateNeed, right: &StateNeed) -> bool { - if shared_size(&left.family, &right.family).is_none() { - return false; - } - let left_inputs = left.build.children(); - let right_inputs = right.build.children(); - if left_inputs.len() != right_inputs.len() - || !left_inputs - .iter() - .zip(right_inputs) - .all(|(a, b)| Rc::ptr_eq(a, b) || same_range_input(a, b)) - { - return false; - } - // Ignore only the fields this rule rewrites. Build guarantees are derived - // from these operators and are recomposed by the existing rewrite helpers. - let signature = |reader: &StateNeed| { - let mut operator = reader.build.operator.map_children(|_| ()); - match &mut operator { - Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) => *family = left.family.clone(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => { - *family = left.family.clone(); - *tumbling_window_width_ms = 1; - *tumbling_window_count = 1; - } - _ => unreachable!(), - } - let mut schema = reader.build.schema.clone(); - for field in &mut schema.fields { - if same_value(&field.dtype, &reader.family) { - field.dtype = left.family.clone(); - } - } - serde_json::to_string(&( - operator, - schema, - reader.build.result_kind, - reader.build.timing, - reader.build.materialization, - )) - .ok() - }; - signature(left).is_some_and(|signature_left| Some(signature_left) == signature(right)) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{entry_with, lower_promql}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - // Request deduplication preserves the entire declared finite configuration - // set while avoiding repeated rewrites through every initial tumbling-window layout. - #[test] - fn equivalent_tumbling_window_layouts_expand_each_configuration_once() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let roots: Vec<_> = [0.95, 0.99] - .into_iter() - .map(|quantile| { - QueryRoot::Operator(lower_promql( - &format!("quantile_over_time({quantile}, latency[5m])"), - target.clone(), - )) - }) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let mut memo = try_populate_pass1(initial, &inventory).unwrap(); - let entries = vec![ - entry_with( - target, - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(6))) - ); - 2 - ]; - crate::pass2::memo::window_candidates::add_local_window_composition(&mut memo, &entries); - let mut unoptimized = memo.clone(); - let optimized_rewrites = expand_configurations(&mut memo, true).unwrap(); - let exhaustive_rewrites = expand_configurations(&mut unoptimized, false).unwrap(); - assert!( - optimized_rewrites < exhaustive_rewrites, - "{optimized_rewrites} versus {exhaustive_rewrites}" - ); - let signatures = |memo: &Memo| -> Vec> { - memo.candidates - .iter() - .map(|candidates| { - candidates - .iter() - .map(|candidate| graph_signature(&candidate.root_node).unwrap()) - .collect() - }) - .collect() - }; - assert_eq!(signatures(&memo), signatures(&unoptimized)); - let widths: HashSet<_> = memo - .candidates - .iter() - .flatten() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some(*tumbling_window_width_ms), - _ => None, - }) - .collect(); - assert_eq!(widths, HashSet::from([1, 2, 3, 6])); - } - - /// The KLL sizes (`k`) of the builds `candidate` reads. - fn kll_sizes(candidate: &asap_types::stages::CandidateForMemoGroup) -> Vec { - OperatorNode::reachable(&candidate.root_node) - .iter() - .filter_map(|node| match node.asap() { - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => match kind.params() { - asap_types::ir::schema::SketchParams::Kll { k, .. } => Some(*k), - _ => None, - }, - _ => None, - }) - .collect() - } - - // A rewrite to a size another reader's build was sized for carries that - // build's shares of the budget: in `q0.5 / q0.9` each operand has a KLL - // sized for the whole budget and one for half of it, and the copy of the - // whole-budget reader rewritten to the half-budget size is sized for two - // shares, not one. - #[test] - fn a_rewrite_to_a_shared_size_carries_its_parts() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.5, latency[5m]) / quantile_over_time(0.9, latency[5m])", - target.clone(), - ))]; - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let mut memo = try_populate_pass1(initial, &inventory).unwrap(); - // Pass 1's parts per KLL size. - let mut parts_of_size: HashMap = HashMap::new(); - for candidate in memo.candidates.iter().flatten() { - for k in kll_sizes(candidate) { - let parts = parts_of_size.entry(k).or_insert(0); - *parts = (*parts).max(candidate.accuracy_budget_parts); - } - } - assert!( - parts_of_size.values().any(|&parts| parts == 2), - "{parts_of_size:?}" - ); - let before = memo.candidates.iter().map(Vec::len).sum::(); - expand_configurations(&mut memo, true).unwrap(); - assert!(memo.candidates.iter().map(Vec::len).sum::() > before); - for candidate in memo.candidates.iter().flatten() { - for k in kll_sizes(candidate) { - if let Some(&parts) = parts_of_size.get(&k) { - assert!( - candidate.accuracy_budget_parts >= parts, - "k {k}: parts {} below {parts}", - candidate.accuracy_budget_parts - ); - } - } - } - } -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_producer.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/shared_producer.rs deleted file mode 100644 index ea01f1956..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_producer.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! The shared-producer rule: readers that need one computation at different -//! values (a top-k `k`, a sketch size, a window and step) share one producer -//! built at a need that covers them, and each reader derives its own result -//! from it as one more candidate; its own candidates stay, so plan selection -//! decides. Identical expressions are the trivial case, shared by memo group -//! identity, and add no candidates here. -use crate::pass2::memo::state_groups::Pass2RuleError; -use asap_types::stages::Memo; - -/// One reader of a kind of shared producer: its memo group and its need. -pub(in crate::pass2::memo) struct Reader { - pub group: usize, - pub need: N, -} - -/// A kind of producer several readers can share. -pub(in crate::pass2::memo) trait SharedProducer { - /// What one reader requires of the producer. - type Need; - /// A producer's need that serves several readers at once. - type Covering; - /// The memo's readers of this kind, in sets whose members can all read - /// one producer; in memo order. - fn sets(&mut self, memo: &Memo) -> Result>>, Pass2RuleError>; - /// The coverings to offer the readers of `set`. - fn coverings(&self, set: &[Reader]) -> Result, Pass2RuleError>; - /// Adds `reader`'s derivation from the producer at `covering`, if that - /// producer serves it; the number of candidates added. - fn derive( - &mut self, - memo: &mut Memo, - reader: &Reader, - covering: &Self::Covering, - ) -> Result; -} - -/// Whether `set`'s readers belong to two or more groups; a set inside one -/// group shares nothing. -pub(in crate::pass2::memo) fn spans_groups(set: &[Reader]) -> bool { - set.iter().any(|reader| reader.group != set[0].group) -} - -/// Gives every reader of a set spanning two or more groups its derivation -/// from each of the set's coverings; the number of candidates added. -pub(in crate::pass2::memo) fn add_shared_producer_candidates( - memo: &mut Memo, - producer: &mut P, -) -> Result { - let mut added = 0; - for set in producer.sets(memo)? { - if !spans_groups(&set) { - continue; - } - let coverings = producer.coverings(&set)?; - for reader in &set { - for covering in &coverings { - added += producer.derive(memo, reader, covering)?; - } - } - } - Ok(added) -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs index e2bb0e84f..956141fbf 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs @@ -1,100 +1,12 @@ //! Expose private value projections and filters below summary builds as canonical inputs. //! This preserves their full expressions and schema, including opaque series //! identity, rather than assuming that a projection is an identity operation. -use super::reuse::{append_producer, append_reader, eligible, replace}; -use crate::pass2::memo::state_groups::{boundary_ptrs, nodes_within, Pass2RuleError}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; -pub(crate) fn factor_summary_inputs(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let originals: Vec<_> = memo - .candidates - .iter() - .enumerate() - .flat_map(|(owner, candidates)| { - candidates - .iter() - .filter(|candidate| eligible(candidate)) - .map(move |candidate| (owner, candidate.clone())) - }) - .collect(); - for (owner, original) in originals { - let boundaries = boundary_ptrs(memo, &original); - let mut inputs = Vec::new(); - for node in nodes_within(&original.root_node, &boundaries) { - if matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) { - for child in node.children() { - if !boundaries.contains(&Rc::as_ptr(child)) { - inputs.push(Rc::clone(child)); - } - } - } - } - let queries = memo.groups[owner].queries.clone(); - let mut replacements = HashMap::new(); - for input in inputs { - canonical_preprocessing(memo, &input, &boundaries, &queries, &mut replacements)?; - } - if !replacements.is_empty() { - let reader = replace(&original.root_node, &replacements); - append_reader(memo, owner, &original, reader, original.origin.clone())?; - } - } - Ok(()) -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -fn canonical_preprocessing( - memo: &mut Memo, - node: &Rc, - boundaries: &HashSet<*const OperatorNode>, - queries: &[usize], - replacements: &mut HashMap<*const OperatorNode, Rc>, -) -> Result { - if replacements.contains_key(&Rc::as_ptr(node)) { - return Ok(true); - } - if boundaries.contains(&Rc::as_ptr(node)) { - // Logical TimeRange groups can remain distinct when their inputs - // differ structurally. The existing state-sharing rule nevertheless - // permits identical range descriptors over the same canonical source. - if let Some(group) = memo.groups.iter().find(|group| { - crate::pass2::memo::state_groups::same_range_input(&group.logical_node, node) - }) { - replacements.insert(Rc::as_ptr(node), Rc::clone(&group.logical_node)); - } - return Ok(true); - } - if !matches!( - node.non_asap(), - Some(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) - ) { - return Ok(false); - } - // Predicates/projections can reference subqueries. Every dependency must - // be canonical, not only the row-stream child. - for child in node.children() { - if !canonical_preprocessing(memo, child, boundaries, queries, replacements)? { - return Ok(false); - } - } - let canonical = replace(node, replacements); - let group = append_producer( - memo, - canonical, - queries, - OriginOfCandidateForMemoGroup::SummaryInput, - // These operators introduce no approximate state. Their identity does - // not depend on how a downstream consumer divides its error budget. - 1, - )?; - replacements.insert( - Rc::as_ptr(node), - Rc::clone(&memo.groups[group.0 as usize].logical_node), - ); - Ok(true) +/// Rule 3, summary inputs: a value projection or filter below a summary build as its own input group. +pub(crate) fn factor_summary_inputs(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 3 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_results.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/summary_results.rs deleted file mode 100644 index 49d84f991..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_results.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! Factor composite implementations at summary readouts. Each readout can then -//! choose a state configuration independently, without enumerating combinations -//! of all private states in the enclosing query. -use super::reuse::{append_producer, append_reader, eligible, replace}; -use crate::pass2::memo::state_groups::{ - boundary_ptrs, direct_input_builds, nodes_within, Pass2RuleError, -}; -use asap_types::ir::{ASAPOp, OperatorNode}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -pub(crate) fn factor_summary_results(memo: &mut Memo) -> Result { - let originals: Vec<_> = memo - .candidates - .iter() - .enumerate() - .flat_map(|(group, candidates)| { - candidates - .iter() - .filter(|candidate| eligible(candidate)) - .map(move |candidate| (group, candidate.clone())) - }) - .collect(); - let mut added = 0; - for (owner, original) in originals { - let builds = direct_input_builds(memo, &original); - if builds.len() < 2 { - continue; - } - let build_pointers: HashSet<_> = - builds.iter().map(|(build, _)| Rc::as_ptr(build)).collect(); - let boundaries = boundary_ptrs(memo, &original); - let mut readouts = Vec::new(); - for node in nodes_within(&original.root_node, &boundaries) { - if !Rc::ptr_eq(&node, &original.root_node) - && matches!( - node.asap(), - Some(ASAPOp::SummaryEstimate { .. } | ASAPOp::FinalizeExactAccumulator { .. }) - ) - { - let private_builds: HashSet<_> = OperatorNode::reachable(&node) - .iter() - .map(Rc::as_ptr) - .filter(|pointer| build_pointers.contains(pointer)) - .collect(); - if private_builds.len() == 1 { - readouts.push(Rc::clone(&node)); - } - } - } - if readouts.is_empty() { - continue; - } - let queries = memo.groups[owner].queries.clone(); - let mut replacements = HashMap::new(); - for readout in readouts { - let group = append_producer( - memo, - Rc::clone(&readout), - &queries, - OriginOfCandidateForMemoGroup::SummaryResult, - original.accuracy_budget_parts, - )?; - // The original reader's allocation was used to generate these - // parameters; lifting the boundary must not reset that provenance. - replacements.insert( - Rc::as_ptr(&readout), - Rc::clone(&memo.groups[group.0 as usize].logical_node), - ); - } - let reader = replace(&original.root_node, &replacements); - added += usize::from(append_reader( - memo, - owner, - &original, - reader, - OriginOfCandidateForMemoGroup::SummaryResult, - )?); - } - Ok(added) -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs index 9593a317e..e3b1b22dc 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs @@ -1,267 +1,11 @@ //! Exact ranking reuse with explicit exact-input groups. No global selection //! flags are required: only exact implementations can supply these inputs. -use super::reuse::{append_producer, append_reader, eligible, replace}; -use super::shared_producer::{add_shared_producer_candidates, Reader, SharedProducer}; -use crate::pass2::asap_aware_cse::topk_reuse_candidates; -use crate::pass2::memo::state_groups::{boundary_ptrs, nodes_within, Pass2RuleError}; -use asap_types::ir::operator::{AggIntent, Reduction}; -use asap_types::ir::properties::AccuracyGuarantee; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::stages::MemoGroupId; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::HashMap; -use std::rc::Rc; -pub(crate) fn add_topk_candidates(memo: &mut Memo) -> Result { - add_shared_producer_candidates( - memo, - &mut TopKRankings { - exact_groups: HashMap::new(), - }, - ) -} - -/// Exact top-k readers of one input and partition with different `k`: the -/// covering need is the ranking with the largest `k`, `Limit(k_max)` over -/// `Sort` of an exact input ([`topk_reuse_candidates`]); each reader derives -/// `Limit(k) ▸ Project` from it. -struct TopKRankings { - /// Each input group's exact-only group ([`exact_input_group`]). - exact_groups: HashMap>, -} - -/// One input's top-k readers' shared ranking: `Limit(k_max)` over `Sort`, -/// and the readers' groups, whose queries the producer serves. -struct SharedRanking { - ranking: Rc, - reader_groups: Vec, -} - -impl SharedProducer for TopKRankings { - /// The ranking of the reader's input and partition, at the largest `k` - /// of its set. - type Need = Rc; - type Covering = SharedRanking; - - fn sets(&mut self, memo: &Memo) -> Result>>>, Pass2RuleError> { - let mut owners = HashMap::new(); - let mut roots = Vec::new(); - for (group, entry) in memo.groups.iter().enumerate() { - let [input] = entry.logical_node.children()[..] else { - continue; - }; - let exact = Rc::new( - (**input) - .clone() - .with_guarantee(Some(AccuracyGuarantee::exact( - "exact input candidate group", - ))), - ); - let Ok(copy) = entry.logical_node.with_new_children(|_| Rc::clone(&exact)) else { - continue; - }; - let copy = Rc::new(copy); - owners.insert(Rc::as_ptr(©), group); - roots.push(QueryRoot::Operator(copy)); - } - Ok(topk_reuse_candidates(&roots) - .into_iter() - .map(|reuse| { - reuse - .members - .iter() - .filter_map(|node| { - Some(Reader { - group: *owners.get(&Rc::as_ptr(node))?, - need: Rc::clone(&reuse.ranking), - }) - }) - .collect() - }) - .collect()) - } - - fn coverings( - &self, - set: &[Reader>], - ) -> Result, Pass2RuleError> { - // The largest `k` holds every smaller top-k, so it serves the set. - Ok(vec![SharedRanking { - ranking: Rc::clone(&set[0].need), - reader_groups: set.iter().map(|reader| reader.group).collect(), - }]) - } - - fn derive( - &mut self, - memo: &mut Memo, - reader: &Reader>, - covering: &SharedRanking, - ) -> Result { - // Interning the ranking is idempotent, so every reader of the set - // reads the producer the first one interned. - let Some(input_group) = memo.groups[covering.reader_groups[0]] - .child_groups - .first() - .copied() - else { - return Ok(0); - }; - let Some(exact_input) = exact_input_group(memo, input_group, &mut self.exact_groups)? - else { - return Ok(0); - }; - let input = Rc::clone(&memo.groups[exact_input.0 as usize].logical_node); - let queries: Vec<_> = covering - .reader_groups - .iter() - .flat_map(|&group| memo.groups[group].queries.iter().copied()) - .collect::>() - .into_iter() - .collect(); - let sort = covering.ranking.children()[0] - .with_new_children(|_| Rc::clone(&input)) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let sort_group = append_producer( - memo, - Rc::new(sort), - &queries, - OriginOfCandidateForMemoGroup::TopKReuse, - 1, - )?; - let sort = Rc::clone(&memo.groups[sort_group.0 as usize].logical_node); - let ranking = covering - .ranking - .with_new_children(|_| Rc::clone(&sort)) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let ranking_group = append_producer( - memo, - Rc::new(ranking), - &queries, - OriginOfCandidateForMemoGroup::TopKReuse, - 1, - )?; - let ranking = Rc::clone(&memo.groups[ranking_group.0 as usize].logical_node); - let owner = reader.group; - let logical = Rc::clone(&memo.groups[owner].logical_node); - let Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(keys), - measures, - .. - }) = logical.non_asap() - else { - return Ok(0); - }; - let [AggIntent::TopK { k, .. }] = measures.as_slice() else { - return Ok(0); - }; - let Some(original) = memo.candidates[owner].iter().find(|candidate| eligible(candidate) && matches!(candidate.root_node.non_asap(), Some(NonASAPOp::Aggregate {measures, ..}) if matches!(measures.as_slice(), [AggIntent::TopK {..}]))).cloned() else { return Ok(0) }; - let limited = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Limit { - n: Some(*k), - offset: 0, - partition_by: keys.clone(), - child: Rc::clone(&ranking), - })) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let columns = asap_types::ir::schema::topk::ranked_row_projection( - &ranking, - &Reduction::Reduce(keys.clone()), - &logical.schema.fields[keys.len()].name, - ) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let reader = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: columns, - qualifier: None, - child: limited, - })) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - Ok(usize::from(append_reader( - memo, - owner, - &original, - reader, - OriginOfCandidateForMemoGroup::TopKReuse, - )?)) - } -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -fn exact_input_group( - memo: &mut Memo, - group: MemoGroupId, - known: &mut HashMap>, -) -> Result, Pass2RuleError> { - if let Some(result) = known.get(&group) { - return Ok(*result); - } - // An in-progress marker also prevents malformed cyclic inventories recursing. - known.insert(group, None); - let originals = memo.candidates[group.0 as usize].clone(); - let original_children: Vec<_> = originals - .iter() - .map(|candidate| candidate.input_groups.clone()) - .collect(); - let mut candidates = Vec::new(); - for mut candidate in originals { - if !eligible(&candidate) { - continue; - } - let boundaries = boundary_ptrs(memo, &candidate); - let mut exact = !nodes_within(&candidate.root_node, &boundaries).iter().any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg {family, ..} | ASAPOp::SummaryTumblingWindowChain {family, ..}) if !matches!(family, FieldDataType::ExactAggregate(..))) - }); - if !exact { - continue; - } - let mut replacements = HashMap::new(); - let mut children = Vec::new(); - for child in &candidate.input_groups { - let Some(exact_child) = exact_input_group(memo, *child, known)? else { - exact = false; - break; - }; - replacements.insert( - Rc::as_ptr(&memo.groups[child.0 as usize].logical_node), - Rc::clone(&memo.groups[exact_child.0 as usize].logical_node), - ); - children.push(exact_child); - } - if !exact { - continue; - } - candidate.root_node = replace(&candidate.root_node, &replacements); - candidate.input_groups = children; - candidates.push(candidate); - } - if candidates.is_empty() { - return Ok(None); - } - if candidates.len() == original_children.len() - && candidates - .iter() - .zip(&original_children) - .all(|(candidate, children)| candidate.input_groups == *children) - { - // An already exact-only group remains the shared source for ordinary - // consumers too; cloning it would unnecessarily split scan/state reuse. - known.insert(group, Some(group)); - return Ok(Some(group)); - } - let node = Rc::new((*candidates[0].root_node).clone().with_guarantee(Some( - AccuracyGuarantee::exact("only exact implementations and exact inputs"), - ))); - let queries = memo.groups[group.0 as usize].queries.clone(); - let id = append_producer( - memo, - node, - &queries, - OriginOfCandidateForMemoGroup::TopKReuse, - 1, - )?; - for candidate in &mut candidates { - candidate.fused_groups = Vec::new(); - } - memo.candidates[id.0 as usize] = candidates; - known.insert(group, Some(id)); - Ok(Some(id)) +/// Rule 4, top-k reuse: a candidate of each top-k group reading a larger-`k` group, with an exact-input requirement. +pub(crate) fn add_topk_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 4 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs b/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs deleted file mode 100644 index 9eb61a2db..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! Finite state configurations shared by memo rules and candidate selection. -use crate::stage0::cse::same_value; -use asap_accuracy::estimators::size_for_all_readers; -use asap_types::ir::schema::{ - sketch::{SketchKind, SketchParams}, - FieldDataType, -}; -use asap_types::workload::greatest_common_divisor; -use std::collections::BTreeSet; - -/// Every tumbling-window width that divides one of `grids` (zero grids -/// ignored), in ascending order. -pub(crate) fn tumbling_window_width_domain(grids: impl IntoIterator) -> Vec { - let grids: BTreeSet = grids.into_iter().filter(|&grid| grid > 0).collect(); - let mut widths = BTreeSet::new(); - for grid in grids { - widths.extend(tumbling_window_width_divisors(grid)); - } - widths.into_iter().collect() -} - -#[derive(Debug, Clone)] -pub(crate) struct TumblingWindowRequirement { - pub tumbling_window_width_ms: u64, - pub window_ms: u64, - pub family: FieldDataType, -} -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct TumblingWindowConfiguration { - pub tumbling_window_width_ms: u64, - pub longest_window_ms: u64, - pub family: FieldDataType, -} -impl TumblingWindowConfiguration { - pub fn serves(&self, reader: &TumblingWindowRequirement) -> bool { - self.tumbling_window_width_ms > 0 - && reader - .tumbling_window_width_ms - .is_multiple_of(self.tumbling_window_width_ms) - && reader - .window_ms - .is_multiple_of(self.tumbling_window_width_ms) - && self - .longest_window_ms - .is_multiple_of(self.tumbling_window_width_ms) - && reader.window_ms <= self.longest_window_ms - && shared_size(&self.family, &reader.family) - .is_some_and(|joined| same_value(&joined, &self.family)) - } -} -/// Every positive integer-millisecond divisor of a grid, in ascending order. -/// The time representation fixes the search resolution; no representative-width -/// sampling or arbitrary candidate cap changes this domain. -pub(crate) fn tumbling_window_width_divisors(grid_ms: u64) -> Vec { - let mut widths = BTreeSet::new(); - let mut divisor = 1; - while divisor <= grid_ms / divisor { - if grid_ms.is_multiple_of(divisor) { - widths.insert(divisor); - widths.insert(grid_ms / divisor); - } - divisor += 1; - } - widths.into_iter().collect() -} - -/// Complete divisor grids, distinct required horizons, and joined state sizes. -/// This enumerates configurations of one compatible producer, not workload plans. -pub(crate) fn tumbling_window_configurations( - readers: &[TumblingWindowRequirement], -) -> Vec { - if readers - .iter() - .any(|r| r.tumbling_window_width_ms == 0 || r.window_ms == 0) - { - return Vec::new(); - } - let grids = - tumbling_window_width_domain(readers.iter().map(|reader| { - greatest_common_divisor(reader.tumbling_window_width_ms, reader.window_ms) - })); - let mut configurations = Vec::new(); - for &tumbling_window_width_ms in &grids { - let eligible: Vec<&TumblingWindowRequirement> = readers - .iter() - .filter(|r| { - r.tumbling_window_width_ms - .is_multiple_of(tumbling_window_width_ms) - && r.window_ms.is_multiple_of(tumbling_window_width_ms) - }) - .collect(); - let windows: BTreeSet = eligible.iter().map(|r| r.window_ms).collect(); - // The sizes: the readers' own, closed under the shared size. - let mut sizes: Vec = Vec::new(); - for reader in &eligible { - if !sizes.iter().any(|known| same_value(known, &reader.family)) { - sizes.push(reader.family.clone()); - } - } - let mut i = 0; - while i < sizes.len() { - for j in 0..i { - if let Some(joined) = shared_size(&sizes[i], &sizes[j]) { - if !sizes.iter().any(|known| same_value(known, &joined)) { - sizes.push(joined); - } - } - } - i += 1; - } - for &longest_window_ms in &windows { - for family in &sizes { - let configuration = TumblingWindowConfiguration { - tumbling_window_width_ms, - longest_window_ms, - family: family.clone(), - }; - let admitted: Vec<&&TumblingWindowRequirement> = eligible - .iter() - .filter(|r| configuration.serves(r)) - .collect(); - let window = admitted.iter().map(|r| r.window_ms).max(); - let size = admitted.iter().try_fold(None::, |acc, r| { - Some(Some(match acc { - None => r.family.clone(), - Some(acc) => shared_size(&acc, &r.family)?, - })) - }); - if window == Some(longest_window_ms) - && size.flatten().is_some_and(|size| same_value(&size, family)) - { - configurations.push(configuration); - } - } - } - } - configurations -} - -/// All distinct joined reader requirements, retaining recorded error contracts. -/// This is configuration generation only; sharing is represented by state groups. -pub(crate) fn state_size_configurations( - requirements: Vec>, -) -> Vec> { - if requirements.iter().any(Option::is_none) { - return vec![None]; - } - let requirements: Vec = requirements.into_iter().flatten().collect(); - let mut joins: Vec = Vec::new(); - for requirement in &requirements { - let mut grown = vec![requirement.clone()]; - for join in &joins { - if let Some(joined) = size_for_all_readers(&[join, requirement]) { - grown.push(joined); - } - } - for join in grown { - if !joins.iter().any(|known| same_size(known, &join)) { - joins.push(join); - } - } - } - let mut configurations = Vec::new(); - for configuration in joins { - let admitted: Vec<_> = requirements - .iter() - .filter(|required| { - size_for_all_readers(&[&configuration, required]) - .is_some_and(|joined| same_size(&joined, &configuration)) - }) - .collect(); - if size_for_all_readers(&admitted).is_some_and(|joined| same_size(&joined, &configuration)) - { - configurations.push(Some(configuration)); - } - } - configurations -} - -fn same_size(a: &SketchParams, b: &SketchParams) -> bool { - a == b && format!("{a:?}") == format!("{b:?}") -} - -pub(crate) fn shared_size(a: &FieldDataType, b: &FieldDataType) -> Option { - match (a, b) { - (FieldDataType::Sketch(a, grouping), FieldDataType::Sketch(b, grouping_b)) - if a.algorithm() == b.algorithm() && grouping == grouping_b => - { - let params = size_for_all_readers(&[a.params(), b.params()])?; - Some(FieldDataType::Sketch( - SketchKind::new(*a.algorithm(), params), - grouping.clone(), - )) - } - (FieldDataType::Sketch(..), _) | (_, FieldDataType::Sketch(..)) => None, - _ => (a == b).then(|| a.clone()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::schema::sketch::{GroupingStrategy, SketchAlgorithm, SketchParams}; - - fn kll(probability: Option) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 331, - failure_probability: probability, - }, - ), - GroupingStrategy::default(), - ) - } - - // State-size generation retains both local and shared capacities without - // depending on the removed implicit coupling-site representation. - #[test] - fn state_sizes_keep_each_distinct_reader_configuration() { - let small = SketchParams::Kll { - k: 100, - failure_probability: None, - }; - let large = SketchParams::Kll { - k: 200, - failure_probability: None, - }; - let configurations = state_size_configurations(vec![ - Some(small.clone()), - Some(large.clone()), - Some(small.clone()), - ]); - assert_eq!(configurations.len(), 2); - assert!(configurations.contains(&Some(small))); - assert!(configurations.contains(&Some(large))); - assert_eq!(state_size_configurations(vec![None, None]), vec![None]); - } - - // Equal capacity does not erase the stricter recorded failure probability. - #[test] - fn state_size_join_preserves_probability_metadata() { - let weak = SketchParams::Kll { - k: 331, - failure_probability: Some(0.01), - }; - let strict = SketchParams::Kll { - k: 331, - failure_probability: Some(0.001), - }; - let configurations = state_size_configurations(vec![Some(weak), Some(strict.clone())]); - assert!(configurations - .iter() - .flatten() - .any(|value| same_size(value, &strict))); - } - - // Equal physical capacities still carry distinct recorded error contracts; - // only the joined contract may serve the stricter probability requirement. - #[test] - fn tumbling_window_configuration_keeps_failure_probability_contract() { - let weaker = TumblingWindowConfiguration { - tumbling_window_width_ms: 1000, - longest_window_ms: 5000, - family: kll(None), - }; - let strict = TumblingWindowRequirement { - tumbling_window_width_ms: 1000, - window_ms: 5000, - family: kll(Some(0.001)), - }; - assert!(!weaker.serves(&strict)); - let configurations = tumbling_window_configurations(&[ - TumblingWindowRequirement { - family: kll(None), - ..strict.clone() - }, - strict.clone(), - ]); - assert!(configurations - .iter() - .any(|configuration| configuration.serves(&strict))); - assert!(configurations - .iter() - .all(|configuration| 1000_u64.is_multiple_of(configuration.tumbling_window_width_ms))); - } - - // Every integer-millisecond width that divides the grid is offered, not - // just the coarsest width or GCDs of the readers' original configurations. - #[test] - fn tumbling_window_width_domain_contains_every_legal_divisor() { - let reader = TumblingWindowRequirement { - tumbling_window_width_ms: 12, - window_ms: 60, - family: kll(None), - }; - let widths: BTreeSet<_> = tumbling_window_configurations(&[reader]) - .into_iter() - .map(|configuration| configuration.tumbling_window_width_ms) - .collect(); - assert_eq!(widths, BTreeSet::from([1, 2, 3, 4, 6, 12])); - } - - // Compare the divisor generator to direct enumeration, including zero. - #[test] - fn width_divisors_match_complete_integer_domain() { - for grid in 0..=256_u64 { - let expected: Vec<_> = (1..=grid) - .filter(|width| grid.is_multiple_of(*width)) - .collect(); - assert_eq!(tumbling_window_width_divisors(grid), expected); - } - } - - // Public finite enumeration rejects invalid grids instead of dividing by zero. - #[test] - fn invalid_tumbling_window_grid_has_no_configurations() { - assert!(tumbling_window_configurations(&[TumblingWindowRequirement { - tumbling_window_width_ms: 0, - window_ms: 5000, - family: kll(None) - }]) - .is_empty()); - } -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs b/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs deleted file mode 100644 index e0c5bf94c..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs +++ /dev/null @@ -1,1615 +0,0 @@ -//! Opt-in concrete summary-state groups for the candidate-connection planner. -//! -//! Simple summary builds and tumbling-window chains directly over canonical input groups -//! gain finite compatible sizes and tumbling-window horizons before exact state interning. -//! Concrete state identity includes capacity, guarantee, update/filter semantics, -//! and materialization. Supported private projections/filters gain explicit input groups. - -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; - -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId}; - -/// Why Pass 2's rules failed on a memo: its precondition check, a rule -/// (2–9), or the shared summary-state groups (10). -#[derive(Debug, thiserror::Error)] -pub enum Pass2RuleError { - #[error("memo group and candidate vectors have different lengths")] - MismatchedGroups, - #[error("memo contains an out-of-range group reference")] - InvalidGroupReference, - #[error("memo group capacity exceeds the group identifier range")] - TooManyGroups, - #[error("memo dependencies contain a cycle")] - CyclicDependencies, - #[error("explicit summary-state groups have already been introduced")] - AlreadyExpanded, - #[error("scan-label candidate expansion: {0}")] - ScanLabelWidening(String), - #[error("state configuration expansion: {0}")] - StateConfiguration(String), - #[error("producer reuse rule: {0}")] - ReuseRule(String), -} - -struct BuildReaders { - build: Rc, - children: Vec, - readers: Vec<(usize, usize, Rc)>, -} - -/// Rejects a memo the Pass 2 rules cannot extend: mismatched or -/// out-of-range group references, shared summary-state groups already -/// added, or cyclic dependencies. Pass 2 checks this before its rules. -pub(crate) fn check_unexpanded_memo(memo: &Memo) -> Result<(), Pass2RuleError> { - validate_references(memo)?; - if memo.groups.iter().any(|group| { - matches!( - group.logical_node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) { - return Err(Pass2RuleError::AlreadyExpanded); - } - if memo.topological_order().is_none() { - return Err(Pass2RuleError::CyclicDependencies); - } - Ok(()) -} - -/// Pass 2's last step, after its rules: give each summary build that two or -/// more candidates make identically its own group, and add to each reader -/// group a candidate reading that group. Every existing candidate stays -/// (unsupported candidate shapes unchanged). Then rebuilds every group's -/// parents and the queries it may serve. On error, discard `memo`: it may -/// hold partial work. -pub(crate) fn add_shared_summary_state_groups(memo: &mut Memo) -> Result<(), Pass2RuleError> { - add_shared_summary_state_groups_with_index(memo, true) -} - -/// `indexed` looks builds up by fingerprint; otherwise by a linear scan -/// (the reference the index is tested against). -fn add_shared_summary_state_groups_with_index( - memo: &mut Memo, - indexed: bool, -) -> Result<(), Pass2RuleError> { - let mut builds: Vec = Vec::new(); - let mut build_index: HashMap, Vec> = HashMap::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - for (build, children) in direct_input_builds(memo, candidate) { - let fingerprint = build_fingerprint(&build); - let bucket = build_index.entry(fingerprint).or_default(); - let matching = if indexed { - bucket - .iter() - .copied() - .find(|&index| same_concrete_build(&builds[index].build, &build)) - } else { - builds - .iter() - .position(|entry| same_concrete_build(&entry.build, &build)) - }; - if let Some(build_index) = matching { - builds[build_index].readers.push((group, index, build)); - } else { - bucket.push(builds.len()); - builds.push(BuildReaders { - build: Rc::clone(&build), - children, - readers: vec![(group, index, build)], - }); - } - } - } - } - let mut state_groups = Vec::new(); - let mut replacements: BTreeMap<(usize, usize), HashMap<*const OperatorNode, Rc>> = - BTreeMap::new(); - for build in builds { - // Repeated private builds within one result are reusable too. The - // reader list records concrete build occurrences, not just owners. - if build.readers.len() < 2 { - continue; - } - let state_group = MemoGroupId( - u32::try_from(memo.groups.len()).map_err(|_| Pass2RuleError::TooManyGroups)?, - ); - let mut queries: Vec<_> = build - .readers - .iter() - .flat_map(|(group, _, _)| memo.groups[*group].queries.iter().copied()) - .collect(); - queries.sort_unstable(); - queries.dedup(); - memo.groups.push(MemoGroup { - logical_node: Rc::clone(&build.build), - child_groups: build.children.clone(), - parents: Vec::new(), - queries, - }); - // The build is its readers' own build, so it is sized for their - // shares of the budget (equal builds have equal sizes). - let accuracy_budget_parts = build - .readers - .iter() - .map(|(group, index, _)| memo.candidates[*group][*index].accuracy_budget_parts) - .max() - .unwrap_or(1); - memo.candidates.push(vec![CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: Rc::clone(&build.build), - input_groups: build.children, - origin: OriginOfCandidateForMemoGroup::SharedBuild, - accuracy_budget_parts, - }]); - for (group, index, own_build) in build.readers { - replacements - .entry((group, index)) - .or_default() - .insert(Rc::as_ptr(&own_build), Rc::clone(&build.build)); - } - state_groups.push(state_group); - } - for ((group, index), replacements) in replacements { - // Factor every available producer boundary together. A producer used by - // only one selected reader is still legal; no state-subset enumeration - // is required. Retain the original entirely local implementation too. - let mut reader = memo.candidates[group][index].clone(); - // Only declared inputs and newly lifted states are boundaries. A - // factored result group's representative can equal its candidate root; - // treating every memo node as an input would create a self-alias here. - let boundaries = reader - .input_groups - .iter() - .chain(&state_groups) - .map(|id| (Rc::as_ptr(&memo.groups[id.0 as usize].logical_node), *id)) - .collect(); - reader.root_node = super::rules::reuse::replace(&reader.root_node, &replacements); - reader.input_groups = boundary_groups(&reader.root_node, &boundaries); - reader.origin = OriginOfCandidateForMemoGroup::SharedBuild; - memo.candidates[group].push(reader); - } - // Parents describe every possible group/candidate dependency, not a selected plan. - for group in &mut memo.groups { - group.parents.clear(); - } - for group in 0..memo.groups.len() { - let dependencies: HashSet<_> = memo.groups[group] - .child_groups - .iter() - .chain( - memo.candidates[group] - .iter() - .flat_map(|candidate| &candidate.input_groups), - ) - .copied() - .collect(); - for child in dependencies { - memo.groups[child.0 as usize] - .parents - .push(MemoGroupId(group as u32)); - } - } - if memo.topological_order().is_none() { - return Err(Pass2RuleError::CyclicDependencies); - } - // The chosen representative input may now serve additional queries through - // state reuse; propagate potential reach along the expanded dependency DAG. - for group in memo - .topological_order() - .ok_or(Pass2RuleError::CyclicDependencies)? - .into_iter() - .rev() - { - let queries = memo.groups[group.0 as usize].queries.clone(); - let children: HashSet<_> = memo.groups[group.0 as usize] - .child_groups - .iter() - .chain( - memo.candidates[group.0 as usize] - .iter() - .flat_map(|candidate| &candidate.input_groups), - ) - .copied() - .collect(); - for child in children { - memo.groups[child.0 as usize].queries.extend(&queries); - memo.groups[child.0 as usize].queries.sort_unstable(); - memo.groups[child.0 as usize].queries.dedup(); - } - } - Ok(()) -} - -fn validate_references(memo: &Memo) -> Result<(), Pass2RuleError> { - if memo.groups.len() != memo.candidates.len() { - return Err(Pass2RuleError::MismatchedGroups); - } - if memo.groups.len() > u32::MAX as usize { - return Err(Pass2RuleError::TooManyGroups); - } - for group in &memo.groups { - if group - .child_groups - .iter() - .chain(&group.parents) - .any(|id| id.0 as usize >= memo.groups.len()) - { - return Err(Pass2RuleError::InvalidGroupReference); - } - } - for candidate in memo.candidates.iter().flatten() { - if candidate - .input_groups - .iter() - .chain(&candidate.fused_groups) - .any(|id| id.0 as usize >= memo.groups.len()) - { - return Err(Pass2RuleError::InvalidGroupReference); - } - } - Ok(()) -} - -pub(super) fn simple_build( - memo: &Memo, - candidate: &CandidateForMemoGroup, -) -> Option<(Rc, Vec)> { - let mut builds = direct_input_builds(memo, candidate); - (builds.len() == 1).then(|| builds.remove(0)) -} - -pub(super) fn direct_input_builds( - memo: &Memo, - candidate: &CandidateForMemoGroup, -) -> Vec<(Rc, Vec)> { - if !matches!( - candidate.origin, - OriginOfCandidateForMemoGroup::Pass1 - | OriginOfCandidateForMemoGroup::TumblingWindows - | OriginOfCandidateForMemoGroup::WidenedScans - | OriginOfCandidateForMemoGroup::Rollup - | OriginOfCandidateForMemoGroup::SummaryResult - ) || candidate.accuracy_budget_parts == 0 - { - return Vec::new(); - } - let boundaries: HashMap<_, _> = candidate - .input_groups - .iter() - .map(|&group| { - ( - Rc::as_ptr(&memo.groups[group.0 as usize].logical_node), - group, - ) - }) - .collect(); - let mut stack = vec![Rc::clone(&candidate.root_node)]; - let mut seen = HashSet::new(); - let mut builds = Vec::new(); - while let Some(node) = stack.pop() { - if boundaries.contains_key(&Rc::as_ptr(&node)) || !seen.insert(Rc::as_ptr(&node)) { - continue; - } - if !Rc::ptr_eq(&node, &candidate.root_node) - && matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) - { - let children: Option> = node - .children() - .iter() - .map(|child| boundaries.get(&Rc::as_ptr(child)).copied()) - .collect(); - if let Some(mut children) = children { - children.dedup(); - builds.push((Rc::clone(&node), children)); - } - } - stack.extend(node.children().into_iter().cloned()); - } - builds -} - -pub(super) fn boundary_groups( - node: &Rc, - boundaries: &HashMap<*const OperatorNode, MemoGroupId>, -) -> Vec { - let mut found = Vec::new(); - let mut stack = vec![Rc::clone(node)]; - let mut seen = HashSet::new(); - while let Some(node) = stack.pop() { - if !seen.insert(Rc::as_ptr(&node)) { - continue; - } - if let Some(&group) = boundaries.get(&Rc::as_ptr(&node)) { - if !found.contains(&group) { - found.push(group); - } - } else { - stack.extend(node.children().into_iter().rev().cloned()); - } - } - found -} - -/// The nodes of `candidate`'s input groups, where its own nodes end. -pub(super) fn boundary_ptrs( - memo: &Memo, - candidate: &CandidateForMemoGroup, -) -> HashSet<*const OperatorNode> { - candidate - .input_groups - .iter() - .map(|group| Rc::as_ptr(&memo.groups[group.0 as usize].logical_node)) - .collect() -} - -/// `root` and the nodes beneath it down to `boundaries` (excluded), each -/// once, in depth-first order (last child first). -pub(super) fn nodes_within( - root: &Rc, - boundaries: &HashSet<*const OperatorNode>, -) -> Vec> { - let mut pending = vec![Rc::clone(root)]; - let mut seen = HashSet::new(); - let mut nodes = Vec::new(); - while let Some(node) = pending.pop() { - if boundaries.contains(&Rc::as_ptr(&node)) || !seen.insert(Rc::as_ptr(&node)) { - continue; - } - pending.extend(node.children().into_iter().cloned()); - nodes.push(node); - } - nodes -} - -// Hashing narrows comparisons only; full strict equality below remains the -// authority, including probability metadata and floating-point corner cases. -#[derive(Hash, PartialEq, Eq)] -struct BuildFingerprint { - local: String, - inputs: Vec<(usize, Option)>, -} - -fn local_fingerprint(node: &OperatorNode) -> Option { - serde_json::to_string(&( - node.operator.map_children(|_| ()), - &node.schema, - node.result_kind, - &node.guarantee, - node.timing, - node.materialization, - )) - .ok() -} - -fn build_fingerprint(node: &OperatorNode) -> Option { - let inputs = node - .children() - .into_iter() - .map(|input| { - match input.non_asap() { - Some(NonASAPOp::TimeRange { child, .. }) => { - // Exactly the existing same_range_input contract: identical - // local range semantics over the same canonical source. - Some((Rc::as_ptr(child) as usize, Some(local_fingerprint(input)?))) - } - _ => Some((Rc::as_ptr(input) as usize, None)), - } - }) - .collect::>>()?; - Some(BuildFingerprint { - local: local_fingerprint(node)?, - inputs, - }) -} - -// Immediate inputs are canonical memo nodes. Compare local fields without -// recursively walking their potentially shared DAGs; serialization additionally -// distinguishes signed-zero parameters, matching conservative CSE semantics. -fn same_concrete_build(left: &OperatorNode, right: &OperatorNode) -> bool { - let left_inputs = left.children(); - let right_inputs = right.children(); - left_inputs.len() == right_inputs.len() - && left_inputs - .iter() - .zip(right_inputs) - .all(|(left, right)| Rc::ptr_eq(left, right) || same_range_input(left, right)) - && same_local_fields(left, right) -} - -// TimeRange lacks a unique row key and therefore is not generally memo-interned. -// Equal ranges over the same canonical input nevertheless feed identical data -// into a summary build, as the existing ASAP-aware summary reuse rule allows. -pub(super) fn same_range_input(left: &OperatorNode, right: &OperatorNode) -> bool { - let ( - Some(NonASAPOp::TimeRange { - child: left_child, .. - }), - Some(NonASAPOp::TimeRange { - child: right_child, .. - }), - ) = (left.non_asap(), right.non_asap()) - else { - return false; - }; - Rc::ptr_eq(left_child, right_child) && same_local_fields(left, right) -} - -fn same_local_fields(left: &OperatorNode, right: &OperatorNode) -> bool { - let signature = |node: &OperatorNode| { - ( - node.operator.map_children(|_| ()), - node.result_kind, - node.schema.clone(), - node.guarantee.clone(), - node.timing, - node.materialization, - ) - }; - let left = signature(left); - let right = signature(right); - left == right - && matches!((serde_json::to_string(&left), serde_json::to_string(&right)), (Ok(left), Ok(right)) if left == right) -} - -#[cfg(test)] -fn replace_node( - node: &Rc, - old: &Rc, - new: &Rc, -) -> Rc { - super::rules::reuse::replace(node, &HashMap::from([(Rc::as_ptr(old), Rc::clone(new))])) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::QueryRoot; - - /// An expanded memo with what its shared summary-state step added. - struct Expanded { - memo: Memo, - /// The groups the shared summary-state step added. - state_groups: Vec, - /// The candidates it added to reader groups. - added_reader_candidates: usize, - } - - /// Pass 2 after local window composition, over the complete tumbling-window-width - /// domain, as the Pass 2 entry runs it. - fn expand(memo: Memo) -> Result { - let memo = crate::test_support::sharing_rules_and_shared_state_groups(memo)?; - Ok(expanded(memo)) - } - - fn expanded(memo: Memo) -> Expanded { - let state_groups = crate::test_support::shared_summary_state_groups(&memo); - // Before this step no candidate is a shared build; after it, each - // state group has one and each added reader candidate is one. - let shared_builds = memo - .candidates - .iter() - .flatten() - .filter(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild) - .count(); - Expanded { - added_reader_candidates: shared_builds - state_groups.len(), - state_groups, - memo, - } - } - - // Hash indexing is only an acceleration: independently run the original - // linear state lookup and compare every generated candidate and state. - #[test] - fn indexed_state_interning_preserves_linear_catalog() { - let memo = with_local_tumbling_windows( - workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[15m])", 0.02), - ("quantile_over_time(0.9, latency{job=\"other\"}[5m])", 0.01), - ]), - &[6, 6, 6], - ); - let mut memo = memo; - crate::pass2::logical_memo_candidates::apply_sharing_rules(&mut memo).unwrap(); - let mut linear = memo.clone(); - let mut indexed = memo; - add_shared_summary_state_groups_with_index(&mut indexed, true).unwrap(); - add_shared_summary_state_groups_with_index(&mut linear, false).unwrap(); - let (indexed, linear) = (expanded(indexed), expanded(linear)); - assert_eq!(indexed.state_groups, linear.state_groups); - assert_eq!(indexed.memo.groups.len(), linear.memo.groups.len()); - for (left, right) in indexed.memo.candidates.iter().zip(&linear.memo.candidates) { - let signatures = |candidates: &[CandidateForMemoGroup]| { - candidates - .iter() - .map(|candidate| { - ( - candidate.input_groups.clone(), - candidate.accuracy_budget_parts, - serde_json::to_string( - &asap_types::ir::flat::flatten(&[QueryRoot::Operator(Rc::clone( - &candidate.root_node, - ))]) - .0, - ) - .unwrap(), - ) - }) - .collect::>() - }; - assert_eq!(signatures(left), signatures(right)); - } - for (left, right) in indexed.memo.groups.iter().zip(&linear.memo.groups) { - assert_eq!(left.child_groups, right.child_groups); - assert_eq!(left.parents, right.parents); - assert_eq!(left.queries, right.queries); - } - } - - fn workload(queries: &[(&str, f64)]) -> Memo { - workload_with_identity(queries, false) - } - - fn workload_with_identity(queries: &[(&str, f64)], identity: bool) -> Memo { - let roots: Vec<_> = queries - .iter() - .map(|(query, error)| { - let root = crate::test_support::lower_promql( - query, - AccuracyTarget::max_error(*error).failure_probability(0.001), - ); - let root = if identity { - asap_types::ir::schema::schema_support::with_promql_series_identity(&root) - .unwrap() - } else { - root - }; - QueryRoot::Operator(root) - }) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - try_populate_pass1(initial, &inventory).unwrap() - } - - // Equal-sized quantiles share one canonical state while local choices remain. - #[test] - fn quantile_readers_share_concrete_state_and_retain_local_candidates() { - let original = workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.01), - ]); - let original_count = original.groups.len(); - let expanded = expand(original.clone()).unwrap(); - assert!(!expanded.state_groups.is_empty()); - assert!(expanded.added_reader_candidates >= 2); - let order = expanded.memo.topological_order().unwrap(); - for state in &expanded.state_groups { - let group = &expanded.memo.groups[state.0 as usize]; - assert_eq!(group.queries, vec![0, 1]); - for input in &group.child_groups { - assert_eq!(expanded.memo.groups[input.0 as usize].queries, vec![0, 1]); - assert!(group.logical_node.children().iter().any(|node| Rc::ptr_eq( - node, - &expanded.memo.groups[input.0 as usize].logical_node - ))); - } - assert_eq!(group.parents.len(), 2); - assert!(!asap_types::stages::root_groups(&expanded.memo.query_roots).contains(state)); - for &parent in &group.parents { - assert!( - order.iter().position(|id| id == state).unwrap() - < order.iter().position(|id| id == &parent).unwrap() - ); - let reader = expanded.memo.candidates[parent.0 as usize] - .iter() - .find(|candidate| candidate.input_groups.contains(state)) - .unwrap(); - assert!(OperatorNode::reachable(&reader.root_node) - .iter() - .any(|node| Rc::ptr_eq(node, &group.logical_node))); - assert_eq!(reader.input_groups, vec![*state]); - } - } - for group in 0..original_count { - for (before, after) in original.candidates[group] - .iter() - .zip(&expanded.memo.candidates[group]) - { - assert!(Rc::ptr_eq(&before.root_node, &after.root_node)); - assert_eq!(before.input_groups, after.input_groups); - } - } - assert!(matches!( - expand(expanded.memo), - Err(Pass2RuleError::AlreadyExpanded) - )); - } - - // A stricter concrete configuration can serve both readers; local sizes remain. - #[test] - fn distinct_sizes_gain_compatible_shared_configurations() { - let original = workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.02), - ]); - let expanded = expand(original.clone()).unwrap(); - assert!(!expanded.state_groups.is_empty()); - for (group, candidates) in original.candidates.iter().enumerate() { - for candidate in candidates { - assert!(expanded.memo.candidates[group] - .iter() - .any(|after| Rc::ptr_eq(&candidate.root_node, &after.root_node))); - } - } - assert!(expanded - .state_groups - .iter() - .any(|id| expanded.memo.groups[id.0 as usize].queries == vec![0, 1])); - } - - // Resizing never makes different source predicates compatible. - #[test] - fn distinct_filters_do_not_share_concrete_builds() { - let original = workload(&[ - ("quantile_over_time(0.95, latency{job=\"a\"}[5m])", 0.01), - ("quantile_over_time(0.99, latency{job=\"b\"}[5m])", 0.02), - ]); - let count = original.groups.len(); - let expanded = expand(original).unwrap(); - assert!(expanded.state_groups.is_empty()); - assert_eq!(expanded.memo.groups.len(), count); - } - - fn with_local_tumbling_windows(mut memo: Memo, grids: &[u64]) -> Memo { - use crate::pass2::asap_aware_cse::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - - let boundaries: HashMap<_, _> = memo - .groups - .iter() - .enumerate() - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - for group in 0..memo.groups.len() { - let grid = grids[memo.groups[group].queries[0]]; - for candidate in memo.candidates[group].clone() { - let roots = vec![QueryRoot::Operator(Rc::clone(&candidate.root_node))]; - let tumbling_windows = tumbling_window_candidates(&roots, &[Some(grid)]); - if tumbling_windows.len() != 1 { - continue; - } - let rewritten = window_composition_pass2_candidates(&roots, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - let QueryRoot::Operator(node) = &rewritten[0] else { - unreachable!() - }; - // Tumbling window rewriting recomposes guarantees; restore the original - // canonical scan boundary rather than making it a private input. - let input = tumbling_windows[0].build.children()[0].children()[0].clone(); - let chain = OperatorNode::reachable(node) - .into_iter() - .find(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - }) - .unwrap(); - let node = replace_node(node, chain.children()[0], &input); - let mut tumbling_window = candidate; - tumbling_window.root_node = Rc::clone(&node); - tumbling_window.input_groups = boundary_groups(&node, &boundaries); - tumbling_window.origin = OriginOfCandidateForMemoGroup::TumblingWindows; - memo.candidates[group].push(tumbling_window); - } - } - memo - } - - // Different label projections can consume one wider tumbling-window state through - // typed reader projections; no original scan needs to contain the union. - #[test] - fn disjoint_scan_labels_share_a_union_tumbling_window_state_with_typed_readers() { - let original = with_local_tumbling_windows( - workload(&[ - ("avg by (job) (quantile_over_time(0.95, latency[5m]))", 0.01), - ( - "avg by (region) (quantile_over_time(0.99, latency[5m]))", - 0.01, - ), - ]), - &[60_000, 60_000], - ); - assert!(!original.groups.iter().any(|group| matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Scan { .. }) - ) && group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == "job") - && group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == "region"))); - let local_nodes: Vec<_> = original - .candidates - .iter() - .map(|candidates| { - candidates - .iter() - .map(|candidate| Rc::clone(&candidate.root_node)) - .collect::>() - }) - .collect(); - let expanded = expand(original).unwrap(); - let shared = expanded - .state_groups - .iter() - .find(|id| { - let group = &expanded.memo.groups[id.0 as usize]; - matches!( - group.logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && group.queries == vec![0, 1] - && ["job", "region"].iter().all(|name| { - group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == *name) - }) - }) - .expect("one state over union labels"); - let state = &expanded.memo.groups[shared.0 as usize]; - assert!(state.parents.len() >= 2); - for parent in &state.parents { - let reader = expanded.memo.candidates[parent.0 as usize] - .iter() - .find(|candidate| candidate.input_groups.contains(shared)) - .unwrap(); - assert_eq!( - reader.root_node.schema.fields, - expanded.memo.groups[parent.0 as usize] - .logical_node - .schema - .fields - ); - assert!(OperatorNode::reachable(&reader.root_node) - .iter() - .any(|node| matches!(node.non_asap(), Some(NonASAPOp::Project { .. })))); - reader.root_node.validate_structure().unwrap(); - } - for (group, locals) in local_nodes.into_iter().enumerate() { - assert!(locals - .iter() - .zip(&expanded.memo.candidates[group]) - .all(|(before, after)| Rc::ptr_eq(before, &after.root_node))); - } - assert!(expanded.memo.topological_order().is_some()); - assert!(state - .child_groups - .iter() - .all(|child| expanded.memo.groups[child.0 as usize].queries == vec![0, 1])); - } - - // Widening shifts identity's position but must retain its full typed field - // and remap the source's unique key rather than treating it as a plain label. - #[test] - fn widened_tumbling_window_inputs_preserve_series_identity_and_keys() { - let memo = with_local_tumbling_windows( - workload_with_identity( - &[ - ("avg by (job) (quantile_over_time(0.95, latency[5m]))", 0.01), - ( - "avg by (region) (quantile_over_time(0.99, latency[5m]))", - 0.01, - ), - ], - true, - ), - &[60_000, 60_000], - ); - let expanded = expand(memo).unwrap(); - let state = expanded - .state_groups - .iter() - .map(|id| &expanded.memo.groups[id.0 as usize]) - .find(|group| { - matches!( - group.logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && group.queries == vec![0, 1] - && ["job", "region"].iter().all(|name| { - group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == *name) - }) - }) - .expect("identity-carrying union tumbling-window state"); - let input = state.logical_node.children()[0]; - assert!(input.schema.has_promql_series_identity()); - let identity = input - .schema - .column_id(asap_types::ir::schema::PROMQL_SERIES_IDENTITY) - .unwrap(); - let time = input.schema.time_index.unwrap(); - assert!(input - .schema - .unique_keys - .iter() - .any(|key| key.contains(&identity) && key.contains(&time))); - for parent in &state.parents { - let reader = expanded.memo.candidates[parent.0 as usize] - .iter() - .find(|candidate| { - candidate.input_groups.contains(&MemoGroupId( - expanded - .memo - .groups - .iter() - .position(|group| Rc::ptr_eq(&group.logical_node, &state.logical_node)) - .unwrap() as u32, - )) - }) - .unwrap(); - assert_eq!( - reader.root_node.schema, - expanded.memo.groups[parent.0 as usize].logical_node.schema - ); - } - } - - /// The groups `group`'s candidates read, directly or through the - /// candidates of the groups they read. - fn groups_read_by(memo: &Memo, group: usize) -> HashSet { - let mut read = HashSet::new(); - let mut pending = vec![group]; - while let Some(group) = pending.pop() { - for input in memo.candidates[group] - .iter() - .flat_map(|candidate| &candidate.input_groups) - { - if read.insert(input.0 as usize) { - pending.push(input.0 as usize); - } - } - } - read - } - - // A scan (or time range) that already has every label of its metric is - // not widened: widening recomposes guarantees, and an unwidened root - // that differs from the original only in its guarantee annotation must - // not become an identity `Project` over its own group. - #[test] - fn scan_label_widening_never_reads_its_own_group() { - for queries in [ - [ - ("sum by (job, region) (m)", 0.01), - ("sum by (job) (m)", 0.01), - ], - [ - ("sum by (job, region) (rate(m[5m]))", 0.01), - ("sum by (job) (rate(m[5m]))", 0.01), - ], - ] { - let mut memo = workload_with_identity(&queries, true); - crate::pass2::logical_memo_candidates::apply_sharing_rules(&mut memo).unwrap(); - assert!( - memo.candidates.iter().flatten().any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::WidenedScans - ), - "{queries:?}: the narrower scan is widened" - ); - for group in 0..memo.groups.len() { - assert!( - !groups_read_by(&memo, group).contains(&group), - "{queries:?}: group {group} reads itself: {:?}", - memo.candidates[group] - .iter() - .map(|candidate| (&candidate.origin, &candidate.input_groups)) - .collect::>() - ); - } - let expanded = expand(workload_with_identity(&queries, true)).unwrap(); - for group in 0..expanded.memo.groups.len() { - assert!(!groups_read_by(&expanded.memo, group).contains(&group)); - } - } - } - - // A candidate reading its own group is a cycle the memo rejects. - #[test] - fn a_group_reading_itself_is_a_cyclic_memo() { - let mut memo = workload(&[("sum by (job) (m)", 0.01)]); - assert!(check_unexpanded_memo(&memo).is_ok()); - let mut reader = memo.candidates[0][0].clone(); - reader.input_groups = vec![MemoGroupId(0)]; - memo.candidates[0].push(reader); - assert_eq!(memo.topological_order(), None); - assert!(matches!( - check_unexpanded_memo(&memo), - Err(Pass2RuleError::CyclicDependencies) - )); - } - - // Label widening must not erase source predicates in order to create reuse. - #[test] - fn scan_label_widening_keeps_different_filters_separate() { - let original = with_local_tumbling_windows( - workload(&[ - ( - "avg by (job) (quantile_over_time(0.95, latency{env=\"a\"}[5m]))", - 0.01, - ), - ( - "avg by (region) (quantile_over_time(0.99, latency{env=\"b\"}[5m]))", - 0.01, - ), - ]), - &[60_000, 60_000], - ); - let expanded = expand(original).unwrap(); - assert!(expanded - .state_groups - .iter() - .all(|id| expanded.memo.groups[id.0 as usize].queries.len() == 1)); - } - - // Real tumbling window rewrites expose one identical chain to distinct quantile readers. - #[test] - fn identical_local_tumbling_window_chains_share_state_without_removing_local_choices() { - let original = with_local_tumbling_windows( - workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.01), - ]), - &[60_000, 60_000], - ); - let local_tumbling_windows = original - .candidates - .iter() - .flatten() - .filter(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::TumblingWindows) - .count(); - assert!(local_tumbling_windows >= 2); - let expanded = expand(original.clone()).unwrap(); - let chains: Vec<_> = expanded - .state_groups - .iter() - .filter(|id| { - matches!( - expanded.memo.groups[id.0 as usize].logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .collect(); - assert!(!chains.is_empty()); - for id in chains { - let group = &expanded.memo.groups[id.0 as usize]; - assert_eq!(group.queries, vec![0, 1]); - assert_eq!(group.parents.len(), 2); - assert!(group.child_groups.iter().all(|input| group - .logical_node - .children() - .iter() - .any(|node| Rc::ptr_eq( - node, - &expanded.memo.groups[input.0 as usize].logical_node - )))); - } - for (group, candidates) in original.candidates.iter().enumerate() { - for original in candidates { - assert!(expanded.memo.candidates[group] - .iter() - .any(|candidate| Rc::ptr_eq(&original.root_node, &candidate.root_node))); - } - } - assert!(matches!( - expand(expanded.memo), - Err(Pass2RuleError::AlreadyExpanded) - )); - } - - // A finer grid and longer horizon serves a shorter reader through newest_tumbling_windows. - #[test] - fn different_tumbling_window_grids_and_horizons_gain_explicit_newest_tumbling_window_readers() { - let original = with_local_tumbling_windows( - workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[15m])", 0.02), - ]), - &[60_000, 30_000], - ); - let expanded = expand(original.clone()).unwrap(); - let state = expanded - .state_groups - .iter() - .find(|id| { - matches!( - expanded.memo.groups[id.0 as usize].logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 30_000, - tumbling_window_count: 30, - .. - }) - ) - }) - .expect("shared fine grid covering fifteen minutes"); - assert_eq!(expanded.memo.groups[state.0 as usize].queries, vec![0, 1]); - assert!(expanded.memo.candidates.iter().flatten().any(|reader| { - reader.input_groups.contains(state) - && OperatorNode::reachable(&reader.root_node) - .iter() - .any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows: Some(10), - .. - }) - ) - }) - })); - for (group, candidates) in original.candidates.iter().enumerate() { - assert!(candidates - .iter() - .all(|before| expanded.memo.candidates[group] - .iter() - .any(|after| Rc::ptr_eq(&before.root_node, &after.root_node)))); - } - assert!(expanded.memo.topological_order().is_some()); - } - - // Rollup readers preserve original outputs while referencing the finer - // query's finalized values through an ordinary child-group dependency: - // the finer query's own result group, whose result is those values. - #[test] - fn exact_rollup_exposes_shared_finalized_values() { - let original = workload(&[ - ("sum by (job) (m)", 0.01), - ("sum by (job, instance) (m)", 0.01), - ]); - let expanded = expand(original).unwrap(); - let memo = &expanded.memo; - let fine = memo.query_roots[1].as_operator().unwrap().0 as usize; - assert!(memo.candidates[fine].iter().all(|candidate| matches!( - candidate.root_node.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - ))); - assert!(memo - .candidates - .iter() - .enumerate() - .filter(|(group, _)| *group != fine) - .flat_map(|(_, candidates)| candidates) - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup - && candidate.input_groups.contains(&MemoGroupId(fine as u32)) - )); - assert!(memo.topological_order().is_some()); - } - - // Sketch rollup exposes the resized finer state through an ordinary group - // dependency, while each output retains its own grouping and quantile. - #[test] - fn sketch_rollup_exposes_shared_finer_state() { - let original = workload(&[ - ("quantile(0.9, m)", 0.01), - ("quantile by (job) (0.9, m)", 0.02), - ]); - let expanded = expand(original).unwrap(); - assert!(expanded - .memo - .candidates - .iter() - .flatten() - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - )) - )); - assert!(expanded.memo.topological_order().is_some()); - } - - // Different k outputs share one sorted ranking, backed by a recursively - // exact candidate group rather than an unenforced exact-input annotation. - #[test] - fn topk_outputs_share_explicit_exact_ranking() { - let original = workload(&[ - ("topk by (job) (3, sum_over_time(m[1m]))", 0.01), - ("topk by (job) (5, sum_over_time(m[1m]))", 0.01), - ]); - let expanded = expand(original).unwrap(); - assert!(expanded.memo.groups.iter().any(|group| matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Limit { n: Some(5), .. }) - ) && group.parents.len() >= 2)); - assert!(expanded - .memo - .candidates - .iter() - .flatten() - .any(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::TopKReuse)); - assert!(expanded.memo.topological_order().is_some()); - } - - // Widened source-group candidates make shared scans available even to - // ordinary operators that do not build an ASAP summary. - #[test] - fn general_source_groups_offer_typed_union_scan_readers() { - let original = workload(&[ - ("topk by (job) (3, m)", 0.01), - ("topk by (region) (5, m)", 0.01), - ]); - let expanded = expand(original.clone()).unwrap(); - let union = expanded - .memo - .groups - .iter() - .enumerate() - .find(|(_, group)| { - matches!(group.logical_node.non_asap(), Some(NonASAPOp::Scan { .. })) - && ["job", "region"].iter().all(|name| { - group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == *name) - }) - }) - .map(|(id, _)| MemoGroupId(id as u32)) - .expect("union scan"); - let readers = original - .groups - .iter() - .enumerate() - .filter(|(_, group)| { - matches!(group.logical_node.non_asap(), Some(NonASAPOp::Scan { .. })) - }) - .filter(|(group, _)| { - expanded.memo.candidates[*group].iter().any(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::WidenedScans - && candidate.input_groups.contains(&union) - }) - }) - .count(); - assert!(readers >= 2); - } - - // A composite average can share only its sum state with another query while - // retaining the count state locally; reader budget allocation is preserved. - #[test] - fn multiple_private_states_can_share_only_the_common_producer() { - let mut original = workload(&[("avg(m)", 0.01), ("sum(m)", 0.01)]); - let average = original.query_roots[0].as_operator().unwrap().0 as usize; - assert!(original.candidates[average] - .iter() - .any(|candidate| direct_input_builds(&original, candidate).len() >= 2)); - for candidates in &mut original.candidates { - for candidate in candidates { - candidate.accuracy_budget_parts = 2; - } - } - let expanded = expand(original.clone()).unwrap(); - assert!(!expanded.state_groups.is_empty()); - let reader = expanded.memo.candidates[average] - .iter() - .find(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && candidate - .input_groups - .iter() - .any(|group| expanded.state_groups.contains(group)) - }) - .expect("average has shared state input"); - assert_eq!(reader.accuracy_budget_parts, 2); - // The shared build is the readers' own, sized for two shares too. - for state in &expanded.state_groups { - assert_eq!( - expanded.memo.candidates[state.0 as usize][0].accuracy_budget_parts, - 2 - ); - } - assert!( - reader - .input_groups - .iter() - .any(|id| expanded.state_groups.contains(id) - && expanded.memo.groups[id.0 as usize].queries == vec![0, 1]), - "sum state is reusable by both queries" - ); - assert!( - reader - .input_groups - .iter() - .any(|id| expanded.state_groups.contains(id) - && expanded.memo.groups[id.0 as usize].queries == vec![0]), - "count state belongs only to the average query" - ); - assert!(original.candidates[average] - .iter() - .all(|before| expanded.memo.candidates[average] - .iter() - .any(|after| Rc::ptr_eq(&before.root_node, &after.root_node)))); - } - - // A global SQL-window quantile can consume newest tumbling windows of a longer, - // finer-grouped chain through the existing regrouping rule. - #[test] - fn grouped_window_rollup_exposes_one_finer_chain() { - use crate::test_support::{agg, scan_from, time_range}; - use asap_types::ir::operator::{AggIntent, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - let scan = || { - scan_from( - Source::Table { - table_ref: "metrics".into(), - }, - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - ) - }; - let quantile = |q| AggIntent::Quantile { - col: Some(2), - q, - accuracy: AccuracyTarget::max_error(0.01), - }; - let roots = vec![ - QueryRoot::Operator(agg( - vec![], - quantile(0.5), - time_range(std::time::Duration::from_secs(300), scan()), - )), - QueryRoot::Operator(agg( - vec![1], - quantile(0.9), - time_range(std::time::Duration::from_secs(1200), scan()), - )), - ]; - let initial = generate_dag_of_logical_memo_groups(&roots); - let generated = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let memo = with_local_tumbling_windows( - try_populate_pass1(initial, &generated).unwrap(), - &[10_000, 10_000], - ); - let expanded = expand(memo).unwrap(); - assert!(expanded - .memo - .candidates - .iter() - .flatten() - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - )) - && candidate.input_groups.iter().any(|group| matches!( - expanded.memo.groups[group.0 as usize].logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count: 120, - .. - }) - )) - )); - } - - // Private identity projections retain their full typed contract, but no - // longer hide compatible builds from canonical state sharing. - #[test] - fn private_projection_inputs_gain_canonical_shared_states() { - check_private_inputs(false); - } - - // Key-only HAVING uses private Filter nodes below summary builds; equal - // filters can share without dropping their predicates. - #[test] - fn private_filter_inputs_gain_canonical_shared_states() { - check_private_inputs(true); - } - - fn check_private_inputs(filter: bool) { - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::{Operator, Predicate, ProjectItem, ScalarExpr}; - let mut memo = workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.01), - ]); - let mut injected = 0; - for owner in 0..memo.candidates.len() { - let originals = memo.candidates[owner].clone(); - for mut candidate in originals { - let builds = direct_input_builds(&memo, &candidate); - if builds.is_empty() { - continue; - } - let mut replacements = HashMap::new(); - for (build, _) in builds { - let input = build.children()[0]; - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: input - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: Rc::clone(input), - })) - .unwrap(); - let project = if filter { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), - child: project, - })) - .unwrap() - } else { - project - }; - replacements.insert(Rc::as_ptr(input), project); - } - candidate.root_node = - crate::pass2::memo::rules::reuse::replace(&candidate.root_node, &replacements); - assert!(direct_input_builds(&memo, &candidate).is_empty()); - memo.candidates[owner].push(candidate); - injected += 1; - } - } - assert!(injected >= 2); - let expanded = expand(memo).unwrap(); - assert!(expanded.state_groups.iter().any(|state| { - let group = &expanded.memo.groups[state.0 as usize]; - group.queries == vec![0, 1] - && group.child_groups.iter().any(|input| { - expanded.memo.candidates[input.0 as usize] - .iter() - .any(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::SummaryInput - }) - }) - })); - assert!(expanded.memo.topological_order().is_some()); - } - - // Real identity-aware frequency candidates retain UnivMon inputs and their - // error metadata when private preprocessing is made an explicit boundary. - #[test] - fn univmon_identity_inputs_remain_admissible_after_factoring() { - use asap_types::ir::schema::{sketch::SketchAlgorithm, FieldDataType}; - let mut memo = workload_with_identity(&[("count_over_time(latency[5m])", 0.01)], true); - let is_univmon = |node: &OperatorNode| matches!(node.asap(), Some(ASAPOp::SummaryAgg {family: FieldDataType::Sketch(kind, _), ..}) if kind.algorithm() == &SketchAlgorithm::UnivMon); - let before: Vec<_> = memo - .candidates - .iter() - .flatten() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .filter(|node| is_univmon(node)) - .collect(); - assert!(!before.is_empty(), "real generation must offer UnivMon"); - crate::pass2::memo::rules::summary_inputs::factor_summary_inputs(&mut memo).unwrap(); - assert!(memo - .candidates - .iter() - .flatten() - .any(|candidate| direct_input_builds(&memo, candidate) - .iter() - .any(|(node, _)| is_univmon(node)))); - for original in before { - assert!( - memo.candidates - .iter() - .flatten() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .any(|node| Rc::ptr_eq(&original, &node)), - "local generated candidates remain intact" - ); - } - } - - // Reader deduplication must retain distinct recorded probability contracts - // even though SketchParams structural equality intentionally ignores them. - #[test] - fn same_capacity_readers_retain_distinct_failure_probability_contracts() { - use asap_types::ir::schema::{ - sketch::{SketchAlgorithm, SketchKind, SketchParams}, - FieldDataType, - }; - let mut memo = workload(&[("quantile_over_time(0.95, latency[5m])", 0.01)]); - let (owner, mut base, build) = memo.candidates.iter().enumerate().find_map(|(owner, candidates)| candidates.iter().find_map(|candidate| { - let (build, _) = simple_build(&memo, candidate)?; - matches!(build.asap(), Some(ASAPOp::SummaryAgg {family: FieldDataType::Sketch(kind, _), ..}) if kind.algorithm() == &SketchAlgorithm::Kll).then(|| (owner, candidate.clone(), build)) - })).unwrap(); - let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, grouping), - .. - }) = build.asap() - else { - unreachable!() - }; - let SketchParams::Kll { k, .. } = kind.params() else { - unreachable!() - }; - let boundaries: HashSet<_> = base - .input_groups - .iter() - .map(|id| Rc::as_ptr(&memo.groups[id.0 as usize].logical_node)) - .collect(); - fn without_private_guarantees( - node: &Rc, - boundaries: &HashSet<*const OperatorNode>, - ) -> Rc { - if boundaries.contains(&Rc::as_ptr(node)) { - return Rc::clone(node); - } - Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| without_private_guarantees(child, boundaries)), - guarantee: None, - ..(**node).clone() - }) - } - let rewritten = |probability| { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: *k, - failure_probability: Some(probability), - }, - ), - grouping.clone(), - ); - let roots = crate::pass2::asap_aware_cse::with_resized_build( - &[QueryRoot::Operator(Rc::clone(&base.root_node))], - &build, - family, - ) - .unwrap(); - let QueryRoot::Operator(root) = &roots[0] else { - unreachable!() - }; - let root = - crate::pass2::memo::rules::reuse::restore_inputs(root, &memo, &base.input_groups); - without_private_guarantees(&root, &boundaries) - }; - let weak = rewritten(0.001); - let strict = rewritten(0.0001); - assert_eq!( - *weak, *strict, - "ordinary equality omits the probability metadata" - ); - base.root_node = weak; - memo.candidates[owner] = vec![base.clone()]; - assert!(crate::pass2::memo::rules::reuse::append_reader( - &mut memo, - owner, - &base, - strict, - OriginOfCandidateForMemoGroup::Rollup - ) - .unwrap()); - assert_eq!(memo.candidates[owner].len(), 2); - } - - // Equal private producers can be shared within one composite result; they - // do not need to originate from different memo owner groups. - #[test] - fn repeated_states_within_one_owner_share_a_producer() { - let mut memo = workload(&[("quantile_over_time(0.95, latency[5m])", 0.01)]); - let (owner, mut candidate) = memo - .candidates - .iter() - .enumerate() - .find_map(|(group, candidates)| { - candidates - .iter() - .find(|candidate| simple_build(&memo, candidate).is_some()) - .map(|candidate| (group, candidate.clone())) - }) - .unwrap(); - let boundaries: HashSet<_> = candidate - .input_groups - .iter() - .map(|id| Rc::as_ptr(&memo.groups[id.0 as usize].logical_node)) - .collect(); - fn duplicate( - node: &Rc, - boundaries: &HashSet<*const OperatorNode>, - ) -> Rc { - if boundaries.contains(&Rc::as_ptr(node)) { - return Rc::clone(node); - } - Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| duplicate(child, boundaries)), - ..(**node).clone() - }) - } - let second = duplicate(&candidate.root_node, &boundaries); - let combined = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![Rc::clone(&candidate.root_node), second], - })) - .unwrap(); - candidate.root_node = Rc::clone(&combined); - memo.groups[owner].logical_node = combined; - memo.candidates[owner] = vec![candidate]; - let expanded = expand(memo).unwrap(); - assert_eq!(expanded.state_groups.len(), 1); - let state = expanded.state_groups[0]; - assert!(expanded.memo.groups[state.0 as usize] - .parents - .contains(&MemoGroupId(owner as u32))); - assert!(expanded.memo.candidates[owner] - .iter() - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && candidate.input_groups.contains(&state) - )); - } - - // Public expansion rejects malformed IDs before indexing dependency arrays. - #[test] - fn malformed_references_fail_before_dependency_traversal() { - let mut memo = workload(&[("quantile_over_time(0.95, latency[5m])", 0.01)]); - memo.groups[0].child_groups.push(MemoGroupId(u32::MAX)); - assert!(matches!( - expand(memo), - Err(Pass2RuleError::InvalidGroupReference) - )); - } -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs b/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs index 468de54d5..a3b660450 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs @@ -1,315 +1,21 @@ -//! Local tumbling-window candidates generated from compatible reader grids. Selected -//! connections, not the union of potential consumers, determine eligibility. -#[cfg(test)] -use super::shared_configurations::tumbling_window_width_divisors; -use super::shared_configurations::tumbling_window_width_domain; -use crate::pass2::asap_aware_cse::{ - tumbling_window_candidates, window_composition_pass2_candidates, -}; -use asap_types::ir::flat::flatten; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId as GroupId}; -use asap_types::workload::QueryWorkloadEntry; -#[cfg(test)] -use std::collections::BTreeSet; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; +//! Rule 1, chain groups (concepts doc, section 3, "Chain groups"): a +//! `SummaryTumblingWindowChain[w](X)` group per width, and a `SummaryMerge` +//! over the `TimeRange` (and `TimeShift`s) over it as a candidate of the +//! summary state group. -/// Offer each supported reader-grid configuration, for every legal -/// tumbling-window width. Stage 2 checks the actual chosen consumers, -/// including ingestion alignment and range evaluation steps. -pub fn add_local_window_composition(memo: &mut Memo, entries: &[QueryWorkloadEntry]) { - // Flatten each existing/generated graph once. Repeatedly flattening the - // growing candidate list makes a complete tumbling-window-width domain quadratic. - let mut known: HashMap<(usize, String), Vec> = HashMap::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - if let Ok(signature) = candidate_signature(candidate) { - known.entry((group, signature)).or_default().push(index); - } - } - } - for (group, candidate) in local_tumbling_window_candidates(memo, entries) { - let Ok(signature) = candidate_signature(&candidate) else { - memo.candidates[group].push(candidate); - continue; - }; - let bucket = known.entry((group, signature)).or_default(); - if !bucket.iter().any(|&index| { - let previous = &memo.candidates[group][index]; - previous.origin == candidate.origin - && previous.input_groups == candidate.input_groups - && previous.fused_groups == candidate.fused_groups - && previous.accuracy_budget_parts == candidate.accuracy_budget_parts - && previous.root_node == candidate.root_node - }) { - bucket.push(memo.candidates[group].len()); - memo.candidates[group].push(candidate); - } - } -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; +use asap_types::workload::QueryWorkloadEntry; -/// Expand legal tumbling-window widths while preserving canonical memo inputs. -fn local_tumbling_window_candidates( - memo: &Memo, +/// Rule 1: for each summary state group that survived dead-candidate +/// elimination, a chain group and a merge candidate per tumbling width $w$ +/// from the gcd closure of the evaluation grids, ranges and offsets of the +/// queries that reach it and repeat on a known grid. `entries[i]` is query +/// `i`'s workload entry. +pub(crate) fn add_local_window_composition( + memo: &mut Memo, entries: &[QueryWorkloadEntry], -) -> Vec<(usize, CandidateForMemoGroup)> { - let group_of = group_of_node(&memo.groups); - let mut own: Vec<(usize, CandidateForMemoGroup)> = Vec::new(); - for (g, group) in memo.groups.iter().enumerate() { - let grids = tumbling_window_width_domain( - group - .queries - .iter() - .filter_map(|&query| entries.get(query)?.evaluation_grid_ms()), - ); - for grid in grids { - for candidate in &memo.candidates[g] { - if !matches!( - candidate.origin, - OriginOfCandidateForMemoGroup::Pass1 - | OriginOfCandidateForMemoGroup::SharedBuild - ) { - continue; - } - let canonical_scans = OperatorNode::reachable(&candidate.root_node) - .into_iter() - .filter(|node| { - matches!( - node.non_asap(), - Some(asap_types::ir::NonASAPOp::Scan { .. }) - ) && group_of.contains_key(&Rc::as_ptr(node)) - }) - .collect::>(); - let roots = [QueryRoot::Operator(Rc::clone(&candidate.root_node))]; - let tumbling_windows = tumbling_window_candidates(&roots, &[Some(grid)]); - if tumbling_windows.is_empty() { - continue; - } - let Ok(pass2_candidates) = - window_composition_pass2_candidates(&roots, &tumbling_windows) - else { - continue; - }; - for pass2_candidate in pass2_candidates { - let Ok(rewritten) = pass2_candidate.roots else { - continue; - }; - let [QueryRoot::Operator(node)] = &rewritten[..] else { - continue; - }; - // Guarantee recomposition can clone unchanged source scans. - // Restore only an unambiguous original scan with the same full - // source/schema/execution contract; scans have intrinsic exact - // semantics. Otherwise the tumbling window's canonical input disappears. - let mut restored = HashMap::new(); - for rewritten in OperatorNode::reachable(node) { - let mut matches = canonical_scans.iter().filter(|original| { - original.operator == rewritten.operator - && original.schema == rewritten.schema - && original.result_kind == rewritten.result_kind - && original.timing == rewritten.timing - && original.materialization == rewritten.materialization - }); - if let Some(original) = matches.next() { - if matches.next().is_none() { - restored.insert(Rc::as_ptr(&rewritten), original); - } - } - } - let node = restore_scan_boundaries(node, &restored, &mut HashMap::new()); - own.push(( - g, - rewritten_candidate( - &group_of, - GroupId(g as u32), - candidate, - node, - OriginOfCandidateForMemoGroup::TumblingWindows, - ), - )); - } - } - } - } - own -} - -fn restore_scan_boundaries( - node: &Rc, - restored: &HashMap<*const OperatorNode, &Rc>, - done: &mut HashMap<*const OperatorNode, Rc>, -) -> Rc { - if let Some(scan) = restored.get(&Rc::as_ptr(node)) { - return Rc::clone(scan); - } - if let Some(previous) = done.get(&Rc::as_ptr(node)) { - return Rc::clone(previous); - } - let mut changed = false; - let operator = node.operator.map_children(|child| { - let rebuilt = restore_scan_boundaries(child, restored, done); - changed |= !Rc::ptr_eq(child, &rebuilt); - rebuilt - }); - let rebuilt = if changed { - Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - Rc::clone(node) - }; - done.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); - rebuilt -} - -// Union of complete divisor domains covers individual readers and every -// compatible subset: common widths belong to each reader's divisor domain. -// Nongridded potential consumers do not veto a repeated reader's tumbling windows. -#[cfg(test)] -fn compatible_reader_grids(group: &MemoGroup, entries: &[QueryWorkloadEntry]) -> Vec { - super::shared_configurations::tumbling_window_width_domain( - group - .queries - .iter() - .filter_map(|&query| entries.get(query)?.evaluation_grid_ms()), - ) -} - -fn group_of_node(groups: &[MemoGroup]) -> HashMap<*const OperatorNode, usize> { - groups - .iter() - .enumerate() - .map(|(g, group)| (Rc::as_ptr(&group.logical_node), g)) - .collect() -} - -/// `from`, a candidate of group `owner`, with root `node` and `origin`. -fn rewritten_candidate( - group_of: &HashMap<*const OperatorNode, usize>, - owner: GroupId, - from: &CandidateForMemoGroup, - node: Rc, - origin: OriginOfCandidateForMemoGroup, -) -> CandidateForMemoGroup { - let implements: Vec = std::iter::once(owner) - .chain(from.fused_groups.iter().copied()) - .collect(); - let reached = |node: &Rc| child_groups(group_of, node, &implements); - let children = if reached(&from.root_node) == from.input_groups { - reached(&node) - } else { - from.input_groups.clone() - }; - CandidateForMemoGroup { - fused_groups: from.fused_groups.clone(), - root_node: node, - input_groups: children, - origin, - accuracy_budget_parts: from.accuracy_budget_parts, - } -} - -fn child_groups( - group_of: &HashMap<*const OperatorNode, usize>, - node: &Rc, - implements: &[GroupId], -) -> Vec { - fn walk( - node: &Rc, - group_of: &HashMap<*const OperatorNode, usize>, - implements: &[GroupId], - seen: &mut HashSet<*const OperatorNode>, - out: &mut Vec, - ) { - for child in node.children() { - if !seen.insert(Rc::as_ptr(child)) { - continue; - } - match group_of.get(&Rc::as_ptr(child)) { - Some(&g) if !implements.contains(&GroupId(g as u32)) => { - if !out.contains(&GroupId(g as u32)) { - out.push(GroupId(g as u32)); - } - } - _ => walk(child, group_of, implements, seen, out), - } - } - } - let mut out = Vec::new(); - walk(node, group_of, implements, &mut HashSet::new(), &mut out); - out -} - -fn candidate_signature(candidate: &CandidateForMemoGroup) -> Result { - serde_json::to_string(&flatten(&[QueryRoot::Operator(Rc::clone(&candidate.root_node))]).0) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{entry_with, lower_promql}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - fn repeated(interval: u32) -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(interval))) - } - - fn fixture(recurrences: &[QueryRecurrence]) -> (Memo, Vec) { - let target = AccuracyTarget::max_error(0.05).failure_probability(0.1); - let root = lower_promql("quantile_over_time(0.95, latency[15m])", target.clone()); - let roots: Vec<_> = recurrences - .iter() - .map(|_| QueryRoot::Operator(Rc::clone(&root))) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let generated = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let memo = try_populate_pass1(initial, &generated).unwrap(); - let entries = recurrences - .iter() - .map(|recurrence| entry_with(target.clone(), recurrence.clone())) - .collect(); - (memo, entries) - } - - // A nongridded potential reader cannot eliminate another reader's tumbling windows. - #[test] - fn nongridded_potential_reader_preserves_local_tumbling_window_candidates() { - let (memo, entries) = fixture(&[repeated(60_000), QueryRecurrence::Unknown]); - let candidates = local_tumbling_window_candidates(&memo, &entries); - assert!(!candidates.is_empty()); - assert!(candidates.iter().all( - |(_, candidate)| candidate.origin == OriginOfCandidateForMemoGroup::TumblingWindows - )); - } - - // Individual grids and subset GCDs allow separate or shared physical choices. - #[test] - fn reader_grids_include_subsets_instead_of_only_full_union() { - let (memo, entries) = fixture(&[ - repeated(120_000), - repeated(180_000), - QueryRecurrence::Unknown, - ]); - let group = &memo.groups[memo.query_roots[0].as_operator().unwrap().0 as usize]; - assert_eq!( - compatible_reader_grids(group, &entries), - tumbling_window_width_divisors(120_000) - .into_iter() - .chain(tumbling_window_width_divisors(180_000)) - .collect::>() - .into_iter() - .collect::>() - ); - } +) -> Result<(), Pass2RuleError> { + let _ = (memo, entries); + todo!("#1207 PR 4: rule 1, chain groups and the merges over them") } diff --git a/crates/logical-asap-optimization/src/stage0.rs b/crates/logical-asap-optimization/src/stage0.rs index c26a9dc2e..fefb25c6c 100644 --- a/crates/logical-asap-optimization/src/stage0.rs +++ b/crates/logical-asap-optimization/src/stage0.rs @@ -1,228 +1,25 @@ -//! Stage 0: the memo groups of the workload's query roots -//! ([`DagOfLogicalMemoGroups`]), by hash-consing identical logical -//! sub-expressions (the `cse` module). Stage 1 starts from these groups. +//! Stage 0: the memo groups of the workload's query roots (a [`Memo`] +//! without candidates). Each node of the queries' DAGs becomes the group of +//! its expression, found or added by [`Memo::find_or_add_group`], so +//! identical sub-expressions are one group. Stage 1 starts from these groups. -/// Structural common-subexpression elimination (hash-consing) over the -/// logical DAG. -pub(crate) mod cse; - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use crate::stage0::cse::share_common_sub_dags; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{DagOfLogicalMemoGroups, MemoGroup, MemoGroupId}; +use asap_types::accuracy::AccuracyTarget; +use asap_types::ir::{QueryRoot, SchemaDerivationError}; +use asap_types::stages::Memo; +use asap_types::workload::DataWorkload; /// Stage 0: the memo groups of the workload's query roots, query `i` -/// being `roots[i]`, by hash-consing identical logical sub-expressions -/// (`share_common_sub_dags`). -pub fn generate_dag_of_logical_memo_groups(roots: &[QueryRoot]) -> DagOfLogicalMemoGroups { - let (groups, group_of) = hash_consed(roots); - let group = |node: &Rc| group_of[&Rc::as_ptr(node)]; - let query_roots = roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryRoot::Operator(group(node)), - QueryRoot::Scalar(expr) => { - QueryRoot::Scalar(expr.map_operator_refs(&mut |node| group(node))) - } - }) - .collect(); - DagOfLogicalMemoGroups::new(groups, query_roots, group_of) -} - -/// Each original node (of `roots`), by address, to its group. -type GroupOf = HashMap<*const OperatorNode, MemoGroupId>; - -/// The groups of `roots`, children before parents, and each original node's -/// group. A scalar query reaches the groups of the operator nodes its -/// expression reads. -fn hash_consed(roots: &[QueryRoot]) -> (Vec, GroupOf) { - let originals: Vec<(usize, Rc)> = roots - .iter() - .enumerate() - .flat_map(|(query, root)| { - operator_roots(root) - .into_iter() - .map(move |node| (query, node)) - }) - .collect(); - let interned = share_common_sub_dags(originals.clone()); - // Number the interned nodes children first. - let mut groups: Vec = Vec::new(); - let mut id_of: HashMap<*const OperatorNode, MemoGroupId> = HashMap::new(); - for (_, root) in &interned { - number(root, &mut groups, &mut id_of); - } - for (query, root) in &interned { - let g = id_of[&Rc::as_ptr(root)]; - let mut reached = HashSet::new(); - reach(&groups, g, &mut reached); - for reached in reached { - let queries = &mut groups[reached.0 as usize].queries; - if !queries.contains(query) { - queries.push(*query); - } - } - } - for group in &mut groups { - group.queries.sort_unstable(); - } - // Interning rebuilds every node over its children in order, so the two - // DAGs are walked side by side. - let mut group_of = GroupOf::new(); - for ((_, original), (_, root)) in originals.iter().zip(&interned) { - pair(original, root, &id_of, &mut group_of); - } - (groups, group_of) -} - -/// The operator nodes a query root reads. -fn operator_roots(root: &QueryRoot) -> Vec> { - root.operator_refs().into_iter().cloned().collect() -} - -/// Number `node` (interned) and its descendants, children first. -fn number( - node: &Rc, - groups: &mut Vec, - id_of: &mut HashMap<*const OperatorNode, MemoGroupId>, -) -> MemoGroupId { - if let Some(&id) = id_of.get(&Rc::as_ptr(node)) { - return id; - } - let children: Vec = node - .children() - .into_iter() - .map(|child| number(child, groups, id_of)) - .collect(); - let id = MemoGroupId(groups.len() as u32); - for &child in &children { - let parents = &mut groups[child.0 as usize].parents; - if !parents.contains(&id) { - parents.push(id); - } - } - groups.push(MemoGroup { - logical_node: Rc::clone(node), - child_groups: children, - parents: Vec::new(), - queries: Vec::new(), - }); - id_of.insert(Rc::as_ptr(node), id); - id -} - -/// `g` and every group below it. -fn reach(groups: &[MemoGroup], g: MemoGroupId, reached: &mut HashSet) { - if reached.insert(g) { - for &child in &groups[g.0 as usize].child_groups { - reach(groups, child, reached); - } - } -} - -/// Map `original` and its descendants to the groups of `interned`, its -/// interned copy. -fn pair( - original: &Rc, - interned: &Rc, - id_of: &HashMap<*const OperatorNode, MemoGroupId>, - group_of: &mut GroupOf, -) { - if group_of - .insert(Rc::as_ptr(original), id_of[&Rc::as_ptr(interned)]) - .is_some() - { - return; - } - for (child, interned_child) in original.children().into_iter().zip(interned.children()) { - pair(child, interned_child, id_of, group_of); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_support::lower_promql; - use asap_types::ir::operator::operator_properties::BinaryOpKind; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; - use asap_types::ir::{NonASAPOp, ScalarExpr}; - use asap_types::stages::queries_rooted_at; - - fn query(text: &str) -> QueryRoot { - QueryRoot::Operator(lower_promql(text, AccuracyTarget::max_error(0.01))) - } - - /// Logical interning shares equivalent results without needing Pass 1 choices. - #[test] - fn repeated_results_share_groups_but_quantile_levels_do_not() { - let roots = vec![ - query("sum by (job) (m)"), - query("sum by (job) (m)"), - query("quantile_over_time(0.95, m[5m])"), - query("quantile_over_time(0.99, m[5m])"), - ]; - let memo = generate_dag_of_logical_memo_groups(&roots); - let root_group = |q: usize| *memo.query_roots[q].as_operator().unwrap(); - assert_eq!(root_group(0), root_group(1)); - assert_ne!(root_group(2), root_group(3)); - for (index, group) in memo.groups.iter().enumerate() { - assert!(group - .child_groups - .iter() - .all(|child| (child.0 as usize) < index)); - for child in &group.child_groups { - assert!(memo.groups[child.0 as usize] - .parents - .contains(&MemoGroupId(index as u32))); - } - } - } - - /// A per-series quantile has no provable unique key, but it is - /// deterministic: both operands of `G2 / G2` read the same rows, so the - /// two equal sub-expressions are one group read twice. - #[test] - fn equal_keyless_deterministic_results_share_one_group() { - let roots = vec![QueryRoot::Operator(lower_promql( - "avg by (job) (quantile_over_time(0.5, data[5m]) / quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m]))", - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ))]; - let memo = generate_dag_of_logical_memo_groups(&roots); - let division = memo - .groups - .iter() - .find(|group| { - matches!(group.logical_node.non_asap(), Some(NonASAPOp::BinaryOp { operator, .. }) - if operator.kind == BinaryOpKind::Arithmetic(ArithmeticOpKind::Div)) - }) - .expect("a `G2 / G3` group"); - assert_eq!( - division.child_groups[0], division.child_groups[1], - "both operands of the division are one group" - ); - assert_eq!(memo.groups.len(), 7); - } - - /// Scalar references are roots of their query; literals need no invented node. - #[test] - fn scalar_and_literal_roots_keep_workload_positions() { - let QueryRoot::Operator(node) = query("sum by (job) (m)") else { - unreachable!() - }; - let roots = vec![ - QueryRoot::Scalar(ScalarExpr::literal_f64(1.0)), - QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(node.clone())), - QueryRoot::Operator(node.clone()), - ]; - let memo = generate_dag_of_logical_memo_groups(&roots); - let id = memo.group_for_node(&node).unwrap(); - assert_eq!(queries_rooted_at(&memo.query_roots, id), vec![1, 2]); - assert_eq!(memo.groups[id.0 as usize].reached_by_queries(), [1, 2]); - assert!(generate_dag_of_logical_memo_groups(&roots[..1]) - .groups - .is_empty()); - } +/// being `roots[i]` with accuracy target `accuracy_targets[i]` (`None`: +/// none), every group without candidates. Each query is added with its +/// $n_q$, the number of values one evaluation returns, which `data` bounds; +/// a `Scan`'s column signs are `data`'s declared ones. Fails with the first +/// node that breaks the node rules over its children's groups +/// ([`Memo::find_or_add_group`]). +pub fn memo_of_queries( + roots: &[QueryRoot], + accuracy_targets: &[Option], + data: &DataWorkload, +) -> Result { + let _ = (roots, accuracy_targets, data); + todo!("#1207 PR 2: Stage 0 over Memo::find_or_add_group, each query with its target and n_q") } diff --git a/crates/logical-asap-optimization/src/stage0/cse.rs b/crates/logical-asap-optimization/src/stage0/cse.rs deleted file mode 100644 index 6f209a4e6..000000000 --- a/crates/logical-asap-optimization/src/stage0/cse.rs +++ /dev/null @@ -1,895 +0,0 @@ -//! Structural common-subexpression elimination over the unified operator IR: -//! bottom-up hash-consing of [`OperatorNode`] DAGs across a workload's roots. -//! -//! CSE only runs on already-bound, already-canonicalized plans — structural -//! matching is meaningless before canonicalization has converged -//! semantically-equivalent queries onto one shape. [`share_common_sub_dags`] -//! is the single entry point, run once per workload batch (a batch of one -//! still deduplicates a query's own repeated sub-DAGs, see below). -//! -//! ## Algorithm: classic hash-consing / value-numbering -//! -//! Bottom-up: every child is interned before its parent, so two parents whose -//! children were independently deduplicated down to the same `Rc`s are -//! structurally identical iff their own fields also match, without re-walking -//! the sub-DAGs. "Child" means everything [`OperatorNode::children`] returns: -//! the operator inputs *and* the operator nodes a scalar expression reads -//! (`PromqlScalarFromVector`, `ScalarSubquery`, `Exists`, `InSubquery`), so a -//! vector read by `scalar(v)` in two queries is shared like any other input. -//! The scalar expressions themselves stay opaque data on their owning node. -//! -//! ## Correctness: hash is a filter, `PartialEq` is the decision -//! -//! This is the one non-negotiable rule. A **false positive** here — two -//! sub-DAGs wrongly judged shareable — is a wrong query answer, not a missed -//! optimization: two different queries would read each other's data. -//! [`structural_hash`] (SipHash over a canonical serialization, no -//! collision-freedom guarantee) may only narrow the candidate set within one -//! bucket; the typed equality check on that bucket ([`same_node`]) is what -//! actually decides sharing, every time, no exceptions for "the hash probably -//! didn't collide." Equality is intentionally conservative: it recognizes -//! *exact* structural matches only, never "a stricter-accuracy summary could -//! also answer a looser request" (that subsumption question belongs to the -//! ASAP matcher, not here). -//! -//! ## Legality -//! -//! Structural equality is necessary but not sufficient. A non-ASAP node is -//! only ever *returned* as a match for another when every execution over the -//! same input rows returns the same output rows -//! ([`NonASAPOp::is_shareable_when_equal`]). Equal nodes are -//! interned children first, so two equal nodes read one input, and one copy -//! returns each reader the rows it would have computed alone. A unique key -//! is not needed: equal per-series quantiles without a provable key are one -//! node. An operator that picks rows by input order alone (an unordered -//! `LIMIT`) is always inserted fresh, because two copies may return -//! different rows. -//! An ASAP node (summary state and its evaluations) has no such gate: equal -//! operator, schema and guarantee make it shareable, exactly as post-ASAP -//! sharing decided before this IR. -//! -//! ## Single-query CSE falls out for free -//! -//! A repeated sub-expression within *one* query (the same grouped aggregate on -//! both `BinaryOp` branches) is deduplicated by the same bottom-up interning — -//! a workload of size one still interns bottom-up within that one DAG. - -use std::collections::hash_map::DefaultHasher; -use std::collections::HashMap; -use std::hash::{Hash, Hasher}; -use std::rc::Rc; - -use asap_types::ir::operator::node::{Operator, OperatorNode}; -use asap_types::ir::NonASAPOp; - -/// [`structural_hash`]'s memoization cache: an already-hashed node's `Rc` -/// pointer to its hash. A fresh cache is always correct; what matters is -/// letting it persist across every node of one bottom-up pass rather than -/// starting a new one per call. The caller must keep every cached node alive -/// for the cache's lifetime, or a reused address would alias a stale entry. -pub(crate) type NodeToStructuralHashCache = HashMap<*const OperatorNode, u64>; - -/// The operator with every child (operator inputs and the operator nodes -/// referenced from its scalar expressions alike) replaced by `()`. What -/// remains is the node's own data: variant tag, scalar expressions, -/// parameters. -fn own_fields(node: &OperatorNode) -> Operator<()> { - node.operator.map_children(|_| ()) -} - -/// Coarse structural hash used only to bucket [`InternTable::intern`]'s -/// candidate search — never the sharing decision ([`same_node`] is). -/// -/// `OperatorNode` carries `f64`s (`ScalarValue::Float64`, quantile targets, -/// `AccuracyGuarantee` bounds, …), so it cannot derive `std::hash::Hash`. The -/// hash is SipHash over two parts: -/// -/// 1. the canonical JSON of [`own_fields`] plus `result_kind`, `schema`, -/// `guarantee`, `timing` and `materialization` — every field `PartialEq` compares except the -/// children. A scalar expression is serialized as data with each operator -/// node it reads replaced by `()`, so a reference to an -/// interned sub-DAG contributes nothing of its own here; -/// 2. for every child in [`OperatorNode::children`] order (operator inputs, -/// then scalar-referenced nodes), the child's own `structural_hash`, -/// memoized in `cache` by `Rc` pointer identity. -/// -/// Part 2 is what makes equal sub-DAGs hash equal whether they are reached -/// through an operator input or through a `scalar(v)`, and what keeps the -/// pass linear: a node is generally a DAG, and re-serializing a shared -/// descendant once per parent would cost `O(sub-DAG)` per node instead of -/// `O(1)` beyond the children's already-known hashes. A non-finite `f64` -/// serializes as `null`, merely widening one (still equality-checked) bucket. -pub(crate) fn structural_hash(node: &OperatorNode, cache: &mut NodeToStructuralHashCache) -> u64 { - fn child_hash(child: &Rc, cache: &mut NodeToStructuralHashCache) -> u64 { - let ptr = Rc::as_ptr(child); - if let Some(&h) = cache.get(&ptr) { - return h; - } - let h = structural_hash(child, cache); - cache.insert(ptr, h); - h - } - - let mut hasher = DefaultHasher::new(); - let own = ( - own_fields(node), - node.result_kind, - &node.schema, - &node.guarantee, - node.timing, - node.materialization, - ); - serde_json::to_string(&own) - .unwrap_or_default() - .hash(&mut hasher); - for child in node.children() { - child_hash(child, cache).hash(&mut hasher); - } - hasher.finish() -} - -/// Numeric `PartialEq` alone conflates signed zeros. The serialized check is -/// additional evidence, never a replacement for typed equality (JSON maps -/// non-finite floats to `null`). Used for the guarantee, whose bounds are -/// floats a shared node must preserve bit-for-bit, and for summary families, -/// whose `SketchParams` equality omits the recorded error and probability. -pub(crate) fn same_value(left: &T, right: &T) -> bool { - left == right - && match (serde_json::to_string(left), serde_json::to_string(right)) { - (Ok(left), Ok(right)) => left == right, - _ => false, - } -} - -/// Cache of child-pair comparisons already decided by [`same_node`], keyed by -/// pointer pair. Only interned (table-owned, hence alive) nodes are keys. -type NodePairToIsSameNodeCache = HashMap<(*const OperatorNode, *const OperatorNode), bool>; - -/// The sharing decision: typed equality of two nodes. -/// -/// `OperatorNode`'s derived `PartialEq` would recurse into children by value -/// even when both sides hold the same `Rc` (`OperatorNode` is not `Eq`, so -/// `Rc` gets no pointer shortcut), expanding a shared diamond once per path. -/// Children are therefore compared by pointer first; only when the pointers -/// differ (an equal child that was not legal to share) are the values -/// compared, memoized per pair so a diamond is still walked once. -fn same_node( - left: &OperatorNode, - right: &OperatorNode, - cache: &mut NodePairToIsSameNodeCache, -) -> bool { - let (lc, rc) = (left.children(), right.children()); - if lc.len() != rc.len() { - return false; - } - let children_equal = lc.iter().zip(&rc).all(|(a, b)| { - if Rc::ptr_eq(a, b) { - return true; - } - let key = (Rc::as_ptr(a), Rc::as_ptr(b)); - if let Some(&eq) = cache.get(&key) { - return eq; - } - let eq = same_node(a, b, cache); - cache.insert(key, eq); - eq - }); - children_equal - && left.result_kind == right.result_kind - && left.schema == right.schema - && left.timing == right.timing - && left.materialization == right.materialization - && same_value(&left.guarantee, &right.guarantee) - && same_value(&own_fields(left), &own_fields(right)) -} - -/// Bottom-up hash-consing table: structurally-equal, sharing-legal nodes -/// collapse onto one `Rc`. -/// -/// `buckets` is keyed by [`structural_hash`] — a coarse candidate filter -/// only. Every entry within one bucket is a full node kept around for the -/// [`same_node`] comparison that actually decides a match; a hash collision -/// between structurally different nodes just means a harmless linear scan of -/// a few extra candidates. -struct InternTable { - buckets: HashMap>>, - /// Persisted for the table's whole lifetime so hashing is `O(1)` per node - /// beyond its children; every cached node is owned by `buckets`. - structural_hash_cache: NodeToStructuralHashCache, - same_node_cache: NodePairToIsSameNodeCache, -} - -impl InternTable { - fn new() -> Self { - Self { - buckets: HashMap::new(), - structural_hash_cache: HashMap::new(), - same_node_cache: HashMap::new(), - } - } - - /// Intern one node whose children are already interned: look it up by - /// [`structural_hash`], confirm with [`same_node`], and — only when - /// sharing is legal (module doc, "Legality") — return the existing `Rc` - /// instead of allocating a new one. - fn intern(&mut self, node: OperatorNode) -> Rc { - let hash = structural_hash(&node, &mut self.structural_hash_cache); - // A node that is not legal to share is never *returned* as a match - // for something else; it still occupies a fresh slot in the bucket - // (harmless: later scans require legality of the new node too). - let reusable = node.is_asap() - || node - .non_asap() - .is_some_and(NonASAPOp::is_shareable_when_equal); - let bucket = self.buckets.entry(hash).or_default(); - if reusable { - if let Some(existing) = bucket - .iter() - .find(|candidate| same_node(candidate, &node, &mut self.same_node_cache)) - { - return Rc::clone(existing); - } - } - let rc = Rc::new(node); - bucket.push(Rc::clone(&rc)); - rc - } -} - -/// Input pointer → (input `Rc`, interned result). The input `Rc` is retained -/// so its address cannot be freed and reused by a fresh allocation while the -/// cache still maps it. -type NodeToInternedNodeCache = HashMap<*const OperatorNode, (Rc, Rc)>; - -/// Intern `node`'s children (recursively), then `node` itself. The rebuilt -/// node keeps `node`'s retained schema, result kind, guarantee, timing and -/// materialization: -/// every child is replaced by an equal node, so each derived property stays -/// valid, and the result is `PartialEq`-equal to the input. -fn intern_bottom_up( - table: &mut InternTable, - visited: &mut NodeToInternedNodeCache, - node: &Rc, -) -> Rc { - if let Some((_, interned)) = visited.get(&Rc::as_ptr(node)) { - return Rc::clone(interned); - } - let operator = node - .operator - .map_children(|child| intern_bottom_up(table, visited, child)); - let rebuilt = OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization); - let interned = table.intern(rebuilt); - visited.insert(Rc::as_ptr(node), (Rc::clone(node), Rc::clone(&interned))); - interned -} - -/// Share structurally-identical, sharing-legal sub-DAGs across a workload's -/// roots (or within one root). Every root's *value* is unchanged -/// (`PartialEq`-equal to its input) — only its internal `Rc` structure may -/// now alias another root's, or another part of its own DAG. A node already -/// reached through two paths is visited once. -/// -/// `Id` is caller-chosen — a workload entry's key, an index, a query name. -pub fn share_common_sub_dags( - roots: Vec<(Id, Rc)>, -) -> Vec<(Id, Rc)> { - let mut table = InternTable::new(); - let mut visited = NodeToInternedNodeCache::new(); - roots - .into_iter() - .map(|(id, root)| (id, intern_bottom_up(&mut table, &mut visited, &root))) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::operator::agg_intent::AggIntent; - use asap_types::ir::operator::asap::ASAPOp; - use asap_types::ir::operator::operator_properties::{ - BinaryOpKind, GroupKeys, Reduction, Source, - }; - use asap_types::ir::properties::guarantee::AccuracyGuarantee; - use asap_types::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; - use asap_types::ir::schema::sketch::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, - }; - use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; - use asap_types::ir::ScalarExpr; - use asap_types::ir::{BinaryOperator, NonASAPOp}; - - use asap_types::ir::properties::AccuracyTarget; - - fn node(op: NonASAPOp) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(op)).unwrap() - } - - /// `[ts, service, value, latency]`, no unique key. - fn scan() -> Rc { - node(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - Field::plain("latency", DataType::Float64, false), - ], - 0, - vec![], - ), - }) - } - - fn quantile_agg(by: Vec, col: Option, q: f64) -> Rc { - node(NonASAPOp::Aggregate { - reduction: Reduction::by(by), - measures: vec![AggIntent::Quantile { - col, - q, - accuracy: AccuracyTarget::exact(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - }) - } - - fn compare(lhs: Rc, rhs: Rc) -> Rc { - node(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Compare(CompareOpKind::Eq), - vector_match: None, - }, - return_bool: false, - lhs, - rhs, - }) - } - - fn two_roots(a: Rc, b: Rc) -> (Rc, Rc) { - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - (Rc::clone(ra), Rc::clone(rb)) - } - - #[test] - fn distinct_column_quantiles_do_not_merge() { - // Grouped (unique key present) so only the differing `col` blocks it. - let (ra, rb) = two_roots( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(3), 0.5), - ); - assert!(!Rc::ptr_eq(&ra, &rb)); - assert_ne!(ra, rb); - } - - // Two aggregates that differ only in one measure's `FILTER` predicate - // compute different values, so structural sharing keeps them apart. - #[test] - fn filtered_and_unfiltered_aggregates_do_not_merge() { - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::Predicate; - let unfiltered = quantile_agg(vec![1], Some(2), 0.5); - let filtered = node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Quantile { - col: Some(2), - q: 0.5, - accuracy: AccuracyTarget::exact(), - }], - output_names: vec![], - filters: vec![Some(Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(3)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), - semantics: asap_types::ir::ExprSemantics::Sql, - }))], - having: None, - child: scan(), - }); - let (ra, rb) = two_roots(unfiltered, filtered); - assert!( - !Rc::ptr_eq(&ra, &rb), - "a filtered measure must not be shared" - ); - assert_ne!(ra, rb); - } - - #[test] - fn identical_global_scalar_aggregates_share_their_single_result() { - let a = quantile_agg(vec![], Some(2), 0.9); - let b = quantile_agg(vec![], Some(2), 0.9); - assert_eq!(a, b, "fixture sanity: structurally equal"); - assert!( - !a.schema.has_unique_key(), - "fixture sanity: a global aggregate has no provable unique key" - ); - let (ra, rb) = two_roots(a, b); - assert!( - Rc::ptr_eq(&ra, &rb), - "a global scalar aggregate has one stable result per evaluation" - ); - } - - /// A global scalar result can be shared only with the same computation. - #[test] - fn distinct_global_quantiles_do_not_merge() { - let (a, b) = two_roots( - quantile_agg(vec![], Some(2), 0.5), - quantile_agg(vec![], Some(2), 0.9), - ); - assert!(!Rc::ptr_eq(&a, &b)); - let (a, b) = two_roots( - quantile_agg(vec![], Some(2), 0.5), - quantile_agg(vec![], Some(3), 0.5), - ); - assert!(!Rc::ptr_eq(&a, &b)); - } - - /// Two equal scans of a keyless table (SQL rows) are one node: each - /// reader reads every row the scan outputs. - #[test] - fn equal_scans_without_a_unique_key_are_shared() { - let table_scan = || { - node(NonASAPOp::Scan { - source: Source::Table { - table_ref: "metrics".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - }) - }; - let a = table_scan(); - assert!(!a.schema.has_unique_key(), "fixture sanity: keyless rows"); - let (ra, rb) = two_roots(a, table_scan()); - assert!(Rc::ptr_eq(&ra, &rb)); - } - - /// Grouped quantiles differing only in accuracy keep two roots over one shared scan. - #[test] - fn differing_accuracy_is_never_merged_but_the_scan_child_is_shared() { - let shared_scan = scan(); - let quantile = |epsilon: f64| { - node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Quantile { - col: Some(2), - q: 0.99, - accuracy: AccuracyTarget::max_error(epsilon), - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(&shared_scan), - }) - }; - let a = quantile(0.01); - assert!(a.schema.has_unique_key(), "fixture sanity: grouped"); - let (ra, rb) = two_roots(a, quantile(0.05)); - assert!(!Rc::ptr_eq(&ra, &rb)); - assert_ne!(ra, rb); - let ( - Some(NonASAPOp::Aggregate { child: child_a, .. }), - Some(NonASAPOp::Aggregate { child: child_b, .. }), - ) = (ra.non_asap(), rb.non_asap()) - else { - panic!("expected Aggregate roots"); - }; - assert!(Rc::ptr_eq(child_a, child_b)); - } - - #[test] - fn median_and_explicit_half_percentile_merge() { - // Two spellings that lower to the identical grouped `Quantile { q: 0.5 }`. - let (m, p) = two_roots( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(2), 0.5), - ); - assert!(Rc::ptr_eq(&m, &p)); - } - - #[test] - fn single_query_shares_its_own_repeated_sub_dag() { - // One root with the same grouped aggregate on both branches, built as - // two separately-allocated sub-DAGs (no sharing yet). - let root = compare( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(2), 0.5), - ); - let shared = share_common_sub_dags(vec![("q", root)]); - let [(_, root)] = shared.as_slice() else { - panic!("expected 1 root"); - }; - let Some(NonASAPOp::BinaryOp { lhs, rhs, .. }) = root.non_asap() else { - panic!("expected BinaryOp root, got {root:?}"); - }; - assert!(Rc::ptr_eq(lhs, rhs)); - } - - #[test] - fn shared_root_value_is_unchanged() { - let a = quantile_agg(vec![1], Some(2), 0.5) - .as_ref() - .clone() - .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))); - let before = Rc::new(a); - let (ra, _) = two_roots(Rc::clone(&before), Rc::clone(&before)); - assert_eq!(ra.as_ref(), before.as_ref()); - assert!( - ra.guarantee.is_some(), - "retained properties survive the rebuild" - ); - } - - // ── scalar-referenced sub-DAGs ────────────────────────────────────── - - /// `vector(scalar(sum by (service) (up)))`. - fn scalar_of_vector() -> Rc { - let sum_up = node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Sum { col: Some(2) }], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - }); - assert!(sum_up.schema.has_unique_key(), "fixture sanity"); - node(NonASAPOp::PromqlVectorFromScalar( - ScalarExpr::PromqlScalarFromVector(sum_up), - )) - } - - fn bridged_vector(root: &Rc) -> &Rc { - match root.non_asap() { - Some(NonASAPOp::PromqlVectorFromScalar(ScalarExpr::PromqlScalarFromVector(v))) => v, - other => panic!("expected vector(scalar(v)), got {other:?}"), - } - } - - #[test] - fn scalar_referenced_vector_is_shared_across_queries() { - let (ra, rb) = two_roots(scalar_of_vector(), scalar_of_vector()); - assert!( - Rc::ptr_eq(bridged_vector(&ra), bridged_vector(&rb)), - "the vector read by scalar(v) is a child and must be interned" - ); - assert!( - Rc::ptr_eq(&ra, &rb), - "the scalar bridge has no unique key but returns the same rows" - ); - } - - #[test] - fn structural_hash_sees_through_a_scalar_reference() { - // Two equal bridges must hash equal whether or not their referenced - // vector is the same Rc — the reference contributes the vector's - // memoized hash, not its identity. - let a = scalar_of_vector(); - let b = scalar_of_vector(); - let mut cache = HashMap::new(); - assert_eq!( - structural_hash(&a, &mut cache), - structural_hash(&b, &mut cache) - ); - assert_eq!( - cache.len(), - 4, - "aggregate + scan cached once per root: {cache:?}" - ); - let other = node(NonASAPOp::PromqlVectorFromScalar( - ScalarExpr::PromqlScalarFromVector(quantile_agg(vec![1], Some(2), 0.5)), - )); - assert_ne!( - structural_hash(&a, &mut cache), - structural_hash(&other, &mut cache) - ); - } - - // ── ASAP nodes ────────────────────────────────────────────────────── - - fn summary_agg(alpha: f64, guarantee: Option) -> Rc { - let family = FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }), - GroupingStrategy::default(), - ); - let schema = Schema::lifted(vec![Field::new("state", family.clone(), false)], None); - assert!(!schema.has_unique_key(), "fixture sanity"); - Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child: scan(), - family, - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - }), - schema, - ) - .with_guarantee(guarantee), - ) - } - - #[test] - fn asap_nodes_share_without_a_unique_key() { - let exact = || Some(AccuracyGuarantee::exact("fixture")); - let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.01, exact())); - assert!(Rc::ptr_eq(&ra, &rb)); - assert!(ra.guarantee.is_some()); - } - - #[test] - fn asap_nodes_with_distinct_parameters_or_guarantees_are_not_shared() { - let exact = || Some(AccuracyGuarantee::exact("fixture")); - let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.001, exact())); - assert!(!Rc::ptr_eq(&ra, &rb), "different sketch parameters"); - let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.01, None)); - assert!( - !Rc::ptr_eq(&ra, &rb), - "an unknown guarantee never borrows an exact one" - ); - assert!(rb.guarantee.is_none()); - } - - #[test] - fn evaluations_share_their_producer_but_not_each_other() { - use asap_types::ir::schema::sketch::SketchStatistic; - let evaluation = |q: f64| { - Rc::new(OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: summary_agg(0.01, None), - query: SketchStatistic::Quantile { q }, - }), - Schema::lifted( - vec![Field::plain("quantile", DataType::Float64, false)], - None, - ), - )) - }; - let (p95, p99) = two_roots(evaluation(0.95), evaluation(0.99)); - let producer = |n: &Rc| Rc::clone(n.children()[0]); - assert!(!Rc::ptr_eq(&p95, &p99)); - assert!(Rc::ptr_eq(&producer(&p95), &producer(&p99))); - } - - // ── structural_hash (DAG-aware memoization) ───────────────────────── - - #[test] - fn structural_hash_is_stable_across_cache_states() { - let agg = quantile_agg(vec![1], Some(2), 0.5); - let mut cold = HashMap::new(); - let mut warm = HashMap::new(); - structural_hash(&scan(), &mut warm); - assert_eq!( - structural_hash(&agg, &mut cold), - structural_hash(&agg, &mut warm), - "hash must be independent of unrelated cache state" - ); - } - - #[test] - fn structural_hash_of_an_internally_shared_dag_matches_the_unshared_equivalent() { - let agg = quantile_agg(vec![1], Some(2), 0.5); - let shared_root = compare(Rc::clone(&agg), Rc::clone(&agg)); - let unshared_root = compare( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(2), 0.5), - ); - assert_eq!( - structural_hash(&shared_root, &mut HashMap::new()), - structural_hash(&unshared_root, &mut HashMap::new()), - ); - } - - #[test] - fn structural_hash_memoizes_a_shared_descendant_exactly_once() { - let agg = quantile_agg(vec![1], Some(2), 0.5); - let root = compare(Rc::clone(&agg), Rc::clone(&agg)); - let mut cache = HashMap::new(); - structural_hash(&root, &mut cache); - assert_eq!( - cache.len(), - 2, - "one entry per unique node in the shared branch (Aggregate + Scan): {cache:?}" - ); - } - - /// `DISTINCT` (every column) returns the same rows each time; a `Dedup` - /// on some columns keeps an unspecified row per key. - #[test] - fn dedup_shares_only_on_every_column() { - let dedup = |cols: Vec| { - node(NonASAPOp::Dedup { - cols, - child: scan(), - }) - }; - let (ra, rb) = two_roots(dedup(vec![]), dedup(vec![])); - assert!(Rc::ptr_eq(&ra, &rb)); - let (ra, rb) = two_roots(dedup(vec![1]), dedup(vec![1])); - assert!(!Rc::ptr_eq(&ra, &rb)); - } - - /// `LIMIT 10` without an order may return any 10 rows, so two copies may - /// differ; one shared copy would make both readers see the same rows. - #[test] - fn unordered_limits_are_not_shared() { - let limit = || { - node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: scan(), - }) - }; - let (ra, rb) = two_roots(limit(), limit()); - assert!(!Rc::ptr_eq(&ra, &rb)); - } - - /// A `LIMIT` over a `Sort` with the same partitions keeps the first rows - /// of a fixed order; over a `Sort` partitioned differently, which rows - /// come first still depends on input order. - #[test] - fn limits_over_a_sort_with_their_partitions_are_shared() { - let limit = |sort_partitions: GroupKeys| { - node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: node(NonASAPOp::Sort { - keys: vec![asap_types::ir::scalar::SortKey { - expr: ScalarExpr::column(2), - ascending: false, - nulls_first: false, - }], - partition_by: sort_partitions, - child: scan(), - }), - }) - }; - let (ra, rb) = two_roots(limit(GroupKeys::default()), limit(GroupKeys::default())); - assert!(Rc::ptr_eq(&ra, &rb)); - let (ra, rb) = two_roots(limit(GroupKeys::by(vec![1])), limit(GroupKeys::by(vec![1]))); - assert!(!Rc::ptr_eq(&ra, &rb)); - } - - /// `count without (service)` has no provable unique key, but it computes - /// the same rows from the same input, so equal copies are one node. - #[test] - fn without_grouping_is_shared() { - let without_agg = || { - node(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![0])), - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - }) - }; - let a = without_agg(); - assert!(!a.schema.has_unique_key()); - let (ra, rb) = two_roots(a, without_agg()); - assert!(Rc::ptr_eq(&ra, &rb)); - } - - #[test] - fn already_shared_nodes_are_visited_once() { - // A diamond already present in the input stays one node and is not - // re-interned per path. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let root = compare(Rc::clone(&agg), Rc::clone(&agg)); - let shared = share_common_sub_dags(vec![("q", root)]); - let Some(NonASAPOp::BinaryOp { lhs, rhs, .. }) = shared[0].1.non_asap() else { - panic!("expected BinaryOp root"); - }; - assert!(Rc::ptr_eq(lhs, rhs)); - assert_eq!(OperatorNode::reachable(&shared[0].1).len(), 3); - } - - // Comparing a shareable node whose equal-but-unshareable children form a - // deep diamond must not expand the diamond once per path. The timeout is - // a coarse runaway guard, not a performance SLA. - #[test] - fn shared_diamond_does_not_expand_during_comparison() { - let (done, completion) = std::sync::mpsc::channel(); - let worker = std::thread::spawn(move || { - fn keyed_diamond() -> Rc { - // An unordered `Limit` at each of the 24 levels is never - // shared, so comparing the `Dedup` on top walks the diamond. - let mut current = scan(); - for _ in 0..24 { - let limited = node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: current, - }); - current = compare(Rc::clone(&limited), limited); - } - node(NonASAPOp::Dedup { - cols: vec![], - child: current, - }) - } - let (ra, rb) = two_roots(keyed_diamond(), keyed_diamond()); - assert!(Rc::ptr_eq(&ra, &rb)); - done.send(()).unwrap(); - }); - completion - .recv_timeout(std::time::Duration::from_secs(5)) - .expect("comparison expanded the shared DAG"); - worker.join().unwrap(); - } - - /// A keyed (hence shareable) projection emitting the literal `value`. - fn keyed_literal(value: f64) -> Rc { - let keyed = node(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - ], - 0, - vec![vec![1]], - ), - }); - node(NonASAPOp::Project { - cols: vec![ - asap_types::ir::ProjectItem { - alias: None, - expr: ScalarExpr::Column(1), - }, - asap_types::ir::ProjectItem { - alias: Some("v".into()), - expr: ScalarExpr::literal_f64(value), - }, - ], - qualifier: None, - child: keyed, - }) - } - - /// Sharing preserves IEEE signed zero, and JSON's `null` encoding of - /// non-finite floats never becomes the equality decision. - #[test] - fn signed_zero_and_nonfinite_values_remain_distinct() { - assert!( - keyed_literal(0.0).schema.has_unique_key(), - "fixture is shareable" - ); - for (a, b) in [ - (0.0, -0.0), - (-0.0, 0.0), - (f64::INFINITY, f64::NEG_INFINITY), - (f64::NAN, f64::NAN), - ] { - let (ra, rb) = two_roots(keyed_literal(a), keyed_literal(b)); - assert!(!Rc::ptr_eq(&ra, &rb), "{a} and {b} must not be shared"); - } - let (ra, rb) = two_roots(keyed_literal(f64::INFINITY), keyed_literal(f64::INFINITY)); - assert!(Rc::ptr_eq(&ra, &rb)); - } -} diff --git a/crates/logical-asap-optimization/src/test_support.rs b/crates/logical-asap-optimization/src/test_support.rs index 988cb4210..1ecaa3654 100644 --- a/crates/logical-asap-optimization/src/test_support.rs +++ b/crates/logical-asap-optimization/src/test_support.rs @@ -4,67 +4,23 @@ use std::rc::Rc; -// Stage 1 internals other crates' tests call: Pass 1's working form and its -// population step, single-candidate composition, Stage 0 hash-consing, and -// the Pass 2 rule entry points those tests build memos with. -pub use crate::pass1::memo_population::try_populate_pass1; -pub use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, PerComputationCandidates, +// Stage 1 internals other crates' tests call: Pass 1's working form. +pub use crate::pass1::computation_targets::{ + generate_per_computation_candidates, PerComputationCandidates, }; -pub use crate::pass2::asap_aware_cse::{ - rollup_candidates, rollup_pass2_candidates, tumbling_window_candidates, - window_composition_pass2_candidates, window_composition_shared_chain_candidates, - with_union_scan_labels, -}; -pub use crate::pass2::memo::window_candidates::add_local_window_composition; -pub use crate::stage0::cse::share_common_sub_dags; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use asap_types::ir::OperatorNode; use asap_types::workload::{ BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, }; -/// Pass 2 after local window composition, on a memo the test composed -/// itself (or not at all): rules 2–9, then -/// `add_shared_summary_state_groups`. -/// Production runs these through -/// [`generate_logical_asap_memo_candidates`](crate::pass2::logical_memo_candidates::generate_logical_asap_memo_candidates). -pub fn sharing_rules_and_shared_state_groups( - mut memo: asap_types::stages::Memo, -) -> Result { - crate::pass2::logical_memo_candidates::apply_sharing_rules(&mut memo)?; - crate::pass2::memo::state_groups::add_shared_summary_state_groups(&mut memo)?; - Ok(memo) -} - -/// The groups `add_shared_summary_state_groups` -/// added to an expanded memo, in order: each has one candidate, the -/// shared build (origin `SharedBuild`); no other group's candidates are -/// only that. -pub fn shared_summary_state_groups( - memo: &asap_types::stages::Memo, -) -> Vec { - memo.candidates - .iter() - .enumerate() - .filter(|(_, candidates)| { - matches!( - candidates.as_slice(), - [only] if only.origin - == asap_types::stages::OriginOfCandidateForMemoGroup::SharedBuild - ) - }) - .map(|(group, _)| asap_types::stages::MemoGroupId(group as u32)) - .collect() -} - /// The statistic classic HLL's failure probability at `precision` and /// `relative_error` over `source` is left as, for Stage 3 tests that bind /// it. The estimator keeps the format crate-private. pub fn hll_classic_failure_statistic(precision: u8, relative_error: f64, source: &str) -> String { - asap_accuracy::estimators::hll::classic_failure_statistic(precision, relative_error, source) + asap_accuracy::guarantee::hll::classic_failure_statistic(precision, relative_error, source) } pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc { @@ -107,13 +63,9 @@ pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc { use std::time::Duration; use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; -use asap_types::ir::properties::timing::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, -}; +use asap_types::ir::operator::arguments::{GroupKeys, Reduction, Source}; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, Predicate, ScalarExpr, TimeRangeKind}; -use asap_types::physical::{Materialization, Medium}; /// A `TimeSeries("m")` scan over `[ts(0), value(1), labels...]`, time index 0, /// no unique key. @@ -220,104 +172,6 @@ pub fn time_range(range: Duration, child: Rc) -> Rc .unwrap() } -/// `root` timed under the default (every sub-DAG `NotMaterialized`) -/// assignment — the shape export and the post-ASAP validators consume. -pub fn timed(root: &Rc) -> Rc { - apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .expect("default materialization timings apply") -} - -/// `root` timed with every sub-DAG `MatAtIngestion` where the DAG allows it. -pub fn at_ingestion(root: &Rc) -> Rc { - apply_materialization_timings( - root, - &MaterializationAssignment::with_default(Materialization::MatAtIngestion(Medium::Memory)), - &mut NodeToTimedNodeCache::new(), - ) - .expect("at-ingestion materialization timings apply") -} - -/// The Count-Min summary of a Stage 1 candidate of `query`. -pub fn cms_summary(query: &str) -> OperatorNode { - use crate::pass1::per_computation_candidates::compose_logical_candidate; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; - use asap_types::ir::{ASAPOp, QueryRoot}; - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let root = lower_promql(query, target); - // Inspect each local computation choice under the built-in deployment. - let per_computation = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&[QueryRoot::Operator(root)]), - ) - .unwrap() - .without_unsupported_summaries(None); - let summary = per_computation - .targets - .iter() - .enumerate() - .flat_map(|(target, choices)| { - let inventory = &per_computation; - (0..choices.choice_count()).filter_map(move |candidate| { - let mut assignment = vec![0; inventory.targets.len()]; - assignment[target] = candidate; - compose_logical_candidate(inventory, &assignment).ok() - }) - }) - .flat_map(|roots| match &roots[0] { - QueryRoot::Operator(node) => OperatorNode::reachable(node), - QueryRoot::Scalar(_) => vec![], - }) - .find(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::Cms - ) - }) - .map(|node| (*node).clone()) - .unwrap_or_else(|| panic!("{query}: a Count-Min summary")); - summary -} - -/// `summary` with its update weight replaced by `weight`. -pub fn weighted( - mut summary: OperatorNode, - weight: asap_types::ir::schema::SummaryInputExpr, -) -> Rc { - use asap_types::ir::{ASAPOp, Operator}; - if let Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) = &mut summary.operator { - input.weight = weight; - } - Rc::new(summary) -} - -/// A point-count estimate over `summary` carrying `guarantee`. -pub fn estimate_over( - summary: Rc, - guarantee: asap_types::ir::properties::AccuracyGuarantee, -) -> Rc { - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::SketchStatistic; - use asap_types::ir::{ASAPOp, Operator}; - Rc::new( - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: summary, - query: SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }, - })) - .unwrap() - .with_guarantee(Some(guarantee)), - ) -} - /// The workload entry of a query evaluated as `recurrence` says, exact and /// with no latency bound. pub fn entry( diff --git a/crates/physical-asap-optimization/Cargo.toml b/crates/physical-asap-optimization/Cargo.toml index 9bc25ba9d..dc4220f5e 100644 --- a/crates/physical-asap-optimization/Cargo.toml +++ b/crates/physical-asap-optimization/Cargo.toml @@ -6,29 +6,7 @@ edition = "2021" # #509 Stage 2, physical ASAP-aware optimization: materialization and # physical operator implementation of each Stage 1 candidate, and the # PhysicalASAPMemoCandidates Stage 3 selects from (`catalog`). Depends only on -# asap-types (the IR Stage 1 emits) and the shared accuracy model -# (asap-accuracy, guarantee recomposition of a selected plan), never on -# Stage 1's crate, plan selection or the executor. Tests build their inputs -# with Stage 1. +# asap-types, never on Stage 1's crate, plan selection or the executor. [dependencies] -asap-accuracy = { path = "../accuracy" } asap-types = { path = "../types" } -# The tumbling-window retention of a realized plan (`AssignmentRealizer::realize`). -asap-cost = { path = "../cost" } -rmp-serde = "1" -serde_json = "1" thiserror = "2" - -[features] -# Exposes `test_support` (single-candidate realization, operator -# implementation, tumbling window coverage, and `test_support::catalog`: catalog -# internals plan selection's unit tests exercise with its solver), for other -# crates' tests. Tests also enumerate whole workloads with the reference pricer. -test-support = ["reference-pricer"] -# `optimize_physical_asap`: the whole-workload enumeration behind the -# developer tools' reference pricing (`asap-devtools`). Planning does not use it. -reference-pricer = [] - -[dev-dependencies] -asap-types = { path = "../types", features = ["test-support"] } -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } diff --git a/crates/physical-asap-optimization/src/catalog/configurations.rs b/crates/physical-asap-optimization/src/catalog/configurations.rs deleted file mode 100644 index c5b8ade0a..000000000 --- a/crates/physical-asap-optimization/src/catalog/configurations.rs +++ /dev/null @@ -1,425 +0,0 @@ -//! Finite, unpriced execution configurations over typed immediate candidates. -//! -//! This does not enumerate consumer subsets. Current summary eligibility is a -//! conjunction of individual-reader restrictions and pairwise interval/phase -//! compatibility; selected-reader constraints encode those conditions. Complete -//! Stage 2 validation still checks operator legality and upstream timing. -//! Potential shared non-summary nodes also receive query materialization; Stage 2 -//! checks highest-shared eligibility on the selected graph, not all possible consumers. -use asap_types::stages::{ProducerCandidates, ProducerSetInterner}; -use std::collections::HashMap; - -use crate::summary_materialization_eligibility; -use asap_types::ir::operator::{agg_has_exact_per_entity_kernel, Reduction}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use super::inventory::CandidateInventory; -use asap_types::ir::QueryRoot; -use asap_types::stages::{ - output_references, MemoGroupId, OperatorNodeOfCandidateForMemoGroupId, - PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalCandidateKey, - PruneReason, PrunedCandidate, ReaderConstraint, -}; - -/// [`PhysicalCandidate::may_keep_across_evaluations_per_sole_reader_query`] for `node` configured -/// with `materialization`: for each query in `entries`, whether -/// `summary_materialization_eligibility` with that query as the only reader -/// allows keeping it across evaluations. Empty unless `node` is a tumbling-window chain, -/// `materialization` is `MatAtQueryKeptAcrossEvaluations(Memory)` and the -/// deployment keeps query-time results. -pub fn may_keep_across_evaluations_per_sole_reader_query( - node: &OperatorNode, - materialization: Materialization, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Vec { - if materialization != Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory) - || !matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - || !query_time_results_retained - { - return Vec::new(); - } - (0..entries.len()) - .map(|reader| { - summary_materialization_eligibility( - node, - &[reader], - entries, - data, - query_time_results_retained, - ) - .is_ok_and(|eligibility| eligibility.may_keep_across_evaluations) - }) - .collect() -} - -#[derive(Debug, thiserror::Error)] -pub enum ConfigurationError { - #[error("invalid source candidate inventory: {0}")] - InvalidInventory(String), - #[error("candidate {candidate} already has timing or materialization assigned")] - AssignedExecution { candidate: usize }, -} - -/// Each operator node of `source` with every timing and materialization -/// Stage 2 allows, one candidate each. `query_roots` are the memo's; their -/// group references are `source`'s output references. -pub fn enumerate_candidate_configurations( - source: &CandidateInventory, - query_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - raw_data_retained: bool, -) -> Result { - let invalid = |message: &str| ConfigurationError::InvalidInventory(message.into()); - source.validate().map_err(|error| invalid(&error))?; - let output_queries: Vec = output_references(query_roots) - .into_iter() - .map(|(query, _)| query) - .collect(); - let source_count = source.operator_nodes_of_candidates_for_memo_groups.len(); - if output_queries.len() != source.query_output_nodes.len() - || output_queries.iter().any(|&query| query >= entries.len()) - { - return Err(invalid("template or output/query mapping is incomplete")); - } - if source - .implementation_roots_of_group - .iter() - .flatten() - .any(|id| id.0 >= source_count) - { - return Err(invalid("group references missing source candidate")); - } - let mut query_outputs = vec![Vec::new(); entries.len()]; - let mut potential = vec![vec![false; entries.len()]; source_count]; - let mut may_produce_query_output = vec![false; source_count]; - for (output, &query) in output_queries.iter().enumerate() { - query_outputs[query].push(output); - let mut pending = source.query_output_nodes[output].clone(); - for root in &pending { - may_produce_query_output[root.0] = true; - } - while let Some(candidate) = pending.pop() { - if !potential[candidate.0][query] { - potential[candidate.0][query] = true; - pending.extend( - source.producer_nodes_per_input[candidate.0] - .iter() - .flat_map(|input| input.iter().copied()), - ); - } - } - } - let mut candidates = Vec::new(); - let mut reader_constraints = Vec::new(); - let mut pruned_candidates = Vec::new(); - let mut source_configurations = vec![Vec::new(); source_count]; - for source_id in 0..source_count { - let node = &source.operator_nodes_of_candidates_for_memo_groups[source_id].physical_node; - if node.timing.is_some() || node.materialization != Materialization::NotMaterialized { - return Err(ConfigurationError::AssignedExecution { - candidate: source_id, - }); - } - let summary = matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ); - let forced_query = matches!( - node.asap(), - Some( - ASAPOp::SummaryEstimate { .. } - | ASAPOp::EvaluatePopulation { .. } - | ASAPOp::MaintainPopulation { .. } - ) - ); - let queries = potential[source_id] - .iter() - .enumerate() - .filter_map(|(query, reached)| reached.then_some(query)) - .collect::>(); - let mut modes = vec![(Materialization::NotMaterialized, ExecutionTiming::QueryTime)]; - if summary { - modes.push(( - Materialization::MatAtQuery(Medium::Memory), - ExecutionTiming::QueryTime, - )); - if queries.iter().any(|&query| { - summary_mode_allowed( - node, - &[query], - Materialization::MatAtIngestion(Medium::Memory), - entries, - data, - query_time_results_retained, - ) - }) { - modes.push(( - Materialization::MatAtIngestion(Medium::Memory), - ExecutionTiming::IngestionTime, - )); - } - if matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - && queries.iter().any(|&query| { - summary_mode_allowed( - node, - &[query], - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - entries, - data, - query_time_results_retained, - ) - }) - { - modes.push(( - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ExecutionTiming::QueryTime, - )); - } - } else if !forced_query { - modes.push(( - Materialization::NotMaterialized, - ExecutionTiming::IngestionTime, - )); - } - // Potential reach only generates the choice. Selected-reader cardinality - // and Stage 2's highest-shared rule determine actual eligibility. - if !summary && queries.len() >= 2 { - modes.push(( - Materialization::MatAtQuery(Medium::Memory), - ExecutionTiming::QueryTime, - )); - } - // Configurations the deployment cannot run are not generated; each is - // recorded as pruned. - let group = source.operator_nodes_of_candidates_for_memo_groups[source_id].group(); - let mut prune_modes = |reason, applies: &dyn Fn(ExecutionTiming) -> bool| { - modes.retain(|&(materialization, timing)| { - let pruned = applies(timing); - if pruned { - pruned_candidates.push(PrunedCandidate { - group, - candidate: PhysicalCandidateKey { - operator_node: OperatorNodeOfCandidateForMemoGroupId(source_id), - timing, - materialization, - }, - reason, - }); - } - !pruned - }); - }; - if !per_entity_aggregate_executable(node) { - prune_modes(PruneReason::PerEntityAggregateNotExecutable, &|_| true); - } - // Without raw data retained, only ingestion-time work reads raw rows. - if !raw_data_retained && matches!(node.operator, Operator::NonASAP(NonASAPOp::Scan { .. })) - { - prune_modes(PruneReason::RawDataNotRetained, &|timing| { - timing != ExecutionTiming::IngestionTime - }); - } - for (materialization, timing) in modes { - let id = PhysicalCandidateId(candidates.len()); - source_configurations[source_id].push(id); - candidates.push(PhysicalCandidate { - operator_node: OperatorNodeOfCandidateForMemoGroupId(source_id), - timing, - materialization, - producer_candidates_per_input: Vec::new(), - may_produce_query_output: may_produce_query_output[source_id] - && timing == ExecutionTiming::QueryTime, - may_keep_across_evaluations_per_sole_reader_query: - may_keep_across_evaluations_per_sole_reader_query( - node, - materialization, - entries, - data, - query_time_results_retained, - ), - }); - if materialization == Materialization::MatAtQuery(Medium::Memory) { - if !summary { - reader_constraints.push(ReaderConstraint::HighestSharedQueryMaterialization { - candidate: id, - query_outputs: query_outputs.clone(), - }); - } - reader_constraints.push(ReaderConstraint::MinimumQueryReaders { - candidate: id, - query_outputs: query_outputs.clone(), - minimum: 2, - }); - } - if summary { - let eligible = queries - .iter() - .copied() - .filter(|&query| { - summary_mode_allowed( - node, - &[query], - materialization, - entries, - data, - query_time_results_retained, - ) - }) - .collect::>(); - let outputs = eligible - .iter() - .flat_map(|&query| query_outputs[query].iter().copied()) - .collect(); - reader_constraints.push(ReaderConstraint::AllowedOutputs { - candidate: id, - outputs, - }); - // The current Stage 2 rules require common intervals/phases, - // whose conflicts are pairwise. No arbitrary set predicate is - // approximated by this encoding. - for (position, &left) in eligible.iter().enumerate() { - for &right in &eligible[position + 1..] { - if !summary_mode_allowed( - node, - &[left, right], - materialization, - entries, - data, - query_time_results_retained, - ) { - for &left_output in &query_outputs[left] { - for &right_output in &query_outputs[right] { - reader_constraints.push( - ReaderConstraint::IncompatibleOutputs { - candidate: id, - left: left_output, - right: right_output, - }, - ); - } - } - } - } - } - } - } - } - let mut producer_sets = ProducerSetInterner::default(); - // Compatibility depends on consumer timing and producer mode/timing only. - // Evaluate each source configuration once per timing, then reuse this exact - // index; reader eligibility remains expressed by separate graph constraints. - let mut timing_index = Vec::with_capacity(source_configurations.len()); - for configurations in &source_configurations { - let mut by_timing = [ProducerCandidates::default(), ProducerCandidates::default()]; - for (timing_index, timing) in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] - .into_iter() - .enumerate() - { - let mut compatible = Vec::new(); - for &producer in configurations { - if compatible_timing(timing, &candidates[producer.0]) { - compatible.push(producer); - } - } - by_timing[timing_index] = producer_sets.intern(compatible); - } - timing_index.push(by_timing); - } - let mut input_cache: HashMap<(ProducerCandidates, ExecutionTiming), ProducerCandidates> = - HashMap::new(); - for candidate in &mut candidates { - let mut requirements = Vec::new(); - for input in &source.producer_nodes_per_input[candidate.operator_node.0] { - let key = (input.clone(), candidate.timing); - let compatible_producers = if let Some(producer_candidates) = input_cache.get(&key) { - producer_candidates.clone() - } else { - let timing = match candidate.timing { - ExecutionTiming::QueryTime => 0, - ExecutionTiming::IngestionTime => 1, - }; - let mut compatible = Vec::new(); - for producer in input { - compatible.extend(timing_index[producer.0][timing].iter().copied()); - } - let producer_candidates = producer_sets.intern(compatible); - input_cache.insert(key, producer_candidates.clone()); - producer_candidates - }; - requirements.push(compatible_producers); - } - candidate.producer_candidates_per_input = requirements; - } - let mut result = PhysicalASAPMemoCandidates::new( - source.operator_nodes_of_candidates_for_memo_groups.clone(), - candidates, - query_roots.to_vec(), - reader_constraints, - ); - result.pruned_candidates = pruned_candidates; - super::validate_candidates(&result).map_err(|error| invalid(&error))?; - Ok(result) -} - -/// Whether the executor can run `node`: false only for an exact per-entity -/// `Aggregate` (a per-series range reduction over raw data samples) that -/// has more than one measure, a measure without an exact per-entity kernel -/// ([`agg_has_exact_per_entity_kernel`]), a measure filter or a `HAVING`. -/// The rule reads one operator node, so Stage 2 does not generate the node -/// rather than Stage 3 rejecting each plan that reads it. -fn per_entity_aggregate_executable(node: &OperatorNode) -> bool { - match &node.operator { - Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures, - filters, - having, - .. - }) => { - matches!(measures.as_slice(), [measure] if agg_has_exact_per_entity_kernel(measure)) - && filters.iter().all(Option::is_none) - && having.is_none() - } - _ => true, - } -} - -pub fn compatible_timing(consumer_timing: ExecutionTiming, producer: &PhysicalCandidate) -> bool { - consumer_timing == producer.timing - || (consumer_timing == ExecutionTiming::QueryTime - && matches!(producer.materialization, Materialization::MatAtIngestion(_))) -} - -fn summary_mode_allowed( - node: &OperatorNode, - readers: &[usize], - mode: Materialization, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> bool { - let Ok(eligibility) = summary_materialization_eligibility( - node, - readers, - entries, - data, - query_time_results_retained, - ) else { - return false; - }; - eligibility.range_compatibility.is_ok() - && match mode { - Materialization::NotMaterialized | Materialization::MatAtQuery(_) => true, - Materialization::MatAtIngestion(_) => eligibility.at_ingestion.is_ok(), - Materialization::MatAtQueryKeptAcrossEvaluations(_) => { - eligibility.may_keep_across_evaluations - } - } -} diff --git a/crates/physical-asap-optimization/src/catalog/factoring.rs b/crates/physical-asap-optimization/src/catalog/factoring.rs deleted file mode 100644 index 4ee9e4b68..000000000 --- a/crates/physical-asap-optimization/src/catalog/factoring.rs +++ /dev/null @@ -1,249 +0,0 @@ -//! Memoize identical producer interfaces and quotient equivalent terminal readers. -//! Internal computations are never merged. -use std::collections::HashMap; - -use asap_types::ir::OperatorNode; -use asap_types::stages::MemoGroupId; - -use asap_types::stages::{CandidateForMemoGroupId, PhysicalCandidateId}; -use asap_types::stages::{ProducerCandidates, ProducerSetInterner}; - -#[derive(Hash, PartialEq, Eq)] -pub struct MemoGroupOutputKey(Vec); - -impl MemoGroupOutputKey { - /// Include everything output_contract_satisfied reads from its expected - /// node. Byte-exact keys preserve parameters that PartialEq or JSON may - /// erase, and include closedness and every required uniqueness key. - pub fn new(group: MemoGroupId, expected: &OperatorNode) -> Result { - rmp_serde::to_vec(&(group.0, &expected.result_kind, &expected.schema)) - .map(Self) - .map_err(|error| error.to_string()) - } -} - -#[derive(Default)] -pub struct MemoGroupOutputToCompatibleProducersCache { - results: HashMap, - lists: ProducerSetInterner, - #[cfg(feature = "test-support")] - disabled: bool, - #[cfg(feature = "test-support")] - hits: std::cell::Cell, -} - -impl MemoGroupOutputToCompatibleProducersCache { - pub fn get(&self, key: &MemoGroupOutputKey) -> Option { - #[cfg(feature = "test-support")] - if self.disabled { - return None; - } - let found = self.results.get(key).cloned(); - #[cfg(feature = "test-support")] - if found.is_some() { - self.hits.set(self.hits.get() + 1); - } - found - } - - #[cfg(feature = "test-support")] - pub fn hits(&self) -> usize { - self.hits.get() - } - - #[cfg(feature = "test-support")] - pub fn disabled() -> Self { - Self { - disabled: true, - ..Self::default() - } - } - - pub fn remember( - &mut self, - key: MemoGroupOutputKey, - producer_candidates: Vec, - ) -> ProducerCandidates { - let producer_candidates = self.lists.intern(producer_candidates); - self.results.insert(key, producer_candidates.clone()); - producer_candidates - } -} - -/// Quotient only terminal unary summary readers. Internal consumers and merge -/// operators retain their original identities. A selection binds the -/// factored reader independently per output reference (one selected node -/// each), so every selection of the unfactored readers maps to one of the -/// factored reader, and each binding of it is an unfactored reader with the -/// same complete interface. -/// -/// These operators bypass ConcreteRows' witness-based row certification. -/// certified_price checks every supplied producer; template-based proposal -/// prices remain advisory. Execution and accuracy are recomputed from selected -/// producers, never from the representative template's child witness. -/// -/// A kept node's `candidates_for_memo_group` becomes the union, ascending, -/// of the candidates of the nodes factored into it, and its -/// `accuracy_budget_parts` the largest of theirs. -pub fn factor_terminal_readers( - inventory: &mut super::inventory::CandidateInventory, -) -> Result { - use super::inventory::CandidateInventoryError; - use asap_types::ir::ASAPOp; - let invalid = |error: String| CandidateInventoryError::InvalidMemo(error); - let count = inventory.operator_nodes_of_candidates_for_memo_groups.len(); - let mut internal = vec![false; count]; - for inputs in &inventory.producer_nodes_per_input { - for input in inputs { - for producer in input { - internal[producer.0] = true; - } - } - } - let mut outputs = vec![Vec::new(); count]; - for (output, root) in inventory.query_output_nodes.iter().enumerate() { - for id in root { - outputs[id.0].push(output); - } - } - let mut classes = HashMap::, usize>::new(); - let mut representative: Vec<_> = (0..count).collect(); - let mut eligible = vec![false; count]; - for id in 0..count { - let node = &inventory.operator_nodes_of_candidates_for_memo_groups[id]; - if internal[id] - || outputs[id].is_empty() - || inventory.producer_nodes_per_input[id].len() != 1 - { - continue; - } - let template = &node.physical_node; - if !matches!( - template.asap(), - Some(ASAPOp::SummaryEstimate { .. } | ASAPOp::FinalizeExactAccumulator { .. }) - ) { - continue; - } - let mut inputs = Vec::new(); - let operator = template.operator.map_children(|child| { - inputs.push(child.clone()); - 0u8 - }); - let [input] = inputs.as_slice() else { - continue; - }; - eligible[id] = true; - let key = rmp_serde::to_vec(&( - node.group().0, - &outputs[id], - operator, - &template.schema, - template.result_kind, - &template.guarantee, - template.timing, - template.materialization, - &input.schema, - input.result_kind, - &input.guarantee, - input.timing, - input.materialization, - )) - .map_err(|error| invalid(error.to_string()))?; - if let Some(&previous) = classes.get(&key) { - representative[id] = previous; - } else { - classes.insert(key, id); - } - } - let removed = representative - .iter() - .enumerate() - .filter(|(id, rep)| *id != **rep) - .count(); - if removed == 0 { - return Ok(0); - } - let mut unions = HashMap::>::new(); - let mut known = HashMap::>::new(); - let mut candidates_for_memo_group = HashMap::>::new(); - let mut accuracy_budget_parts = HashMap::::new(); - for (id, &rep) in representative.iter().enumerate() { - if eligible[id] { - let producer_candidates = unions.entry(rep).or_default(); - let known = known.entry(rep).or_default(); - for &producer_id in &inventory.producer_nodes_per_input[id][0] { - if known.insert(producer_id) { - producer_candidates.push(producer_id); - } - } - } - if id != rep { - let factored = &inventory.operator_nodes_of_candidates_for_memo_groups[id]; - candidates_for_memo_group - .entry(rep) - .or_default() - .extend(&factored.candidates_for_memo_group); - let parts = accuracy_budget_parts.entry(rep).or_insert(0); - *parts = (*parts).max(factored.accuracy_budget_parts); - } - } - let mut new_ids = vec![PhysicalCandidateId(0); count]; - let mut next = 0; - for (id, &rep) in representative.iter().enumerate() { - if id == rep { - new_ids[id] = PhysicalCandidateId(next); - next += 1; - } - } - for (id, &rep) in representative.iter().enumerate() { - new_ids[id] = new_ids[rep]; - } - let mut lists = ProducerSetInterner::default(); - let mut nodes = Vec::with_capacity(next); - let mut producer_nodes_per_input = Vec::with_capacity(next); - for (id, node) in inventory - .operator_nodes_of_candidates_for_memo_groups - .iter() - .enumerate() - { - if representative[id] != id { - continue; - } - let mut node = node.clone(); - if let Some(factored) = candidates_for_memo_group.remove(&id) { - node.candidates_for_memo_group.extend(factored); - node.candidates_for_memo_group.sort_unstable(); - node.candidates_for_memo_group.dedup(); - } - if let Some(parts) = accuracy_budget_parts.remove(&id) { - node.accuracy_budget_parts = node.accuracy_budget_parts.max(parts); - } - let mut inputs = inventory.producer_nodes_per_input[id].clone(); - if let Some(producer_candidates) = unions.remove(&id) { - inputs[0] = producer_candidates.into(); - } - for input in &mut inputs { - *input = lists.intern(input.iter().map(|id| new_ids[id.0]).collect::>()); - } - nodes.push(node); - producer_nodes_per_input.push(inputs); - } - let remap = |ids: &mut Vec| { - let mut seen = std::collections::HashSet::new(); - *ids = ids - .iter() - .map(|id| new_ids[id.0]) - .filter(|id| seen.insert(*id)) - .collect(); - }; - for root in &mut inventory.query_output_nodes { - remap(root); - } - for group in &mut inventory.implementation_roots_of_group { - remap(group); - } - inventory.operator_nodes_of_candidates_for_memo_groups = nodes; - inventory.producer_nodes_per_input = producer_nodes_per_input; - inventory.validate().map_err(invalid)?; - Ok(removed) -} diff --git a/crates/physical-asap-optimization/src/catalog/inventory.rs b/crates/physical-asap-optimization/src/catalog/inventory.rs deleted file mode 100644 index cc8daf559..000000000 --- a/crates/physical-asap-optimization/src/catalog/inventory.rs +++ /dev/null @@ -1,346 +0,0 @@ -//! Lower the logical candidate memo into typed immediate operator nodes and -//! the nodes that may produce each of their inputs. -//! -//! Consumer-dependent demand, composed accuracy, and physical materialization -//! are handled by later configuration and evaluation stages. Private exact -//! operators are lowered before decomposition; the resulting nodes still need -//! Stage 2 validation before execution. Nothing is priced. - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::OperatorNode; -use asap_types::physical::Materialization; -use asap_types::stages::Memo; -use asap_types::stages::MemoGroupId; - -use super::factoring::{MemoGroupOutputKey, MemoGroupOutputToCompatibleProducersCache}; -use asap_types::stages::{ - CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, PhysicalCandidateId, - ProducerCandidates, -}; - -/// The memo's implementation candidates lowered to operator nodes, before -/// timing and materialization. Node IDs index -/// `operator_nodes_of_candidates_for_memo_groups`; they become -/// [`asap_types::stages::OperatorNodeOfCandidateForMemoGroupId`]s of the -/// configured candidates. -#[derive(Debug)] -pub struct CandidateInventory { - /// One per operator node: each implementation root, then the private - /// nodes below it. - pub operator_nodes_of_candidates_for_memo_groups: Vec, - /// Per node, per input (in `map_children` order): the nodes that may - /// produce it (a private child, or the input group's compatible roots). - pub producer_nodes_per_input: Vec>, - /// Per output reference: the roots of its group whose output matches the - /// group's logical node. - pub query_output_nodes: Vec>, - /// Per group: its implementation roots, excluding each implementation's - /// private nodes. - pub implementation_roots_of_group: Vec>, -} - -impl CandidateInventory { - /// Every producer and query output names an existing node, once per list. - pub fn validate(&self) -> Result<(), String> { - let count = self.operator_nodes_of_candidates_for_memo_groups.len(); - if self.producer_nodes_per_input.len() != count { - return Err("producer lists are not indexed by node".into()); - } - for (node, inputs) in self.producer_nodes_per_input.iter().enumerate() { - for (input, producer_candidates) in inputs.iter().enumerate() { - super::validate_ids( - producer_candidates, - count, - &format!("candidate {node}, requirement {input}"), - )?; - } - } - for (output, nodes) in self.query_output_nodes.iter().enumerate() { - super::validate_ids(nodes, count, &format!("query root {output}"))?; - } - Ok(()) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum CandidateInventoryError { - #[error("invalid candidate memo: {0}")] - InvalidMemo(String), - #[error("unsupported candidate {candidate} in group {group}: {reason}")] - Unsupported { - group: usize, - candidate: usize, - reason: &'static str, - }, - #[error( - "physical operator implementation for candidate {candidate} in group {group}: {reason}" - )] - PhysicalImplementation { - group: usize, - candidate: usize, - reason: String, - }, -} - -/// The inventory with identical terminal readers factored into one node -/// (whose `candidates_for_memo_group` lists every candidate it can implement). -pub fn build_candidate_inventory( - memo: &Memo, - output_groups: &[MemoGroupId], -) -> Result { - let mut inventory = build_with_compatible_producers_cache( - memo, - output_groups, - &mut MemoGroupOutputToCompatibleProducersCache::default(), - )?; - super::factoring::factor_terminal_readers(&mut inventory)?; - Ok(inventory) -} - -pub fn build_with_compatible_producers_cache( - memo: &Memo, - output_groups: &[MemoGroupId], - compatible_producers_cache: &mut MemoGroupOutputToCompatibleProducersCache, -) -> Result { - let invalid = |message: &str| CandidateInventoryError::InvalidMemo(message.into()); - if memo.groups.len() != memo.candidates.len() { - return Err(invalid("group and candidate vector lengths differ")); - } - if output_groups - .iter() - .any(|group| group.0 as usize >= memo.groups.len()) - { - return Err(invalid("query output references missing group")); - } - let mut inventory = CandidateInventory { - operator_nodes_of_candidates_for_memo_groups: Vec::new(), - producer_nodes_per_input: Vec::new(), - query_output_nodes: Vec::new(), - implementation_roots_of_group: vec![Vec::new(); memo.groups.len()], - }; - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - let unsupported = |reason| CandidateInventoryError::Unsupported { - group, - candidate: index, - reason, - }; - let implemented: HashSet<_> = std::iter::once(MemoGroupId(group as u32)) - .chain(candidate.fused_groups.iter().copied()) - .collect(); - if implemented.len() != 1 + candidate.fused_groups.len() - || implemented - .iter() - .any(|id| id.0 as usize >= memo.groups.len()) - { - return Err(invalid( - "candidate names duplicate or missing absorbed groups", - )); - } - // Only the owner group is this candidate's output contract. Its fused - // groups describe absorbed computations inside its private subtree, - // not additional results it can produce. Private-node decomposition - // charges that work; independent consumers of an absorbed group - // still select one of that group's own candidates. Reverse support - // leaves unused absorbed groups inactive without exclusion rules. - if candidate.accuracy_budget_parts == 0 { - return Err(unsupported("zero accuracy budget parts")); - } - if candidate - .input_groups - .iter() - .any(|child| child.0 as usize >= memo.groups.len()) - { - return Err(invalid("candidate references missing input group")); - } - let inputs = candidate - .input_groups - .iter() - .map(|child| Rc::clone(&memo.groups[child.0 as usize].logical_node)) - .collect::>(); - if inputs - .iter() - .any(|input| Rc::ptr_eq(input, &candidate.root_node)) - { - return Err(unsupported( - "root aliases an input group without an operator", - )); - } - let implemented = crate::implement_candidate_operators(&candidate.root_node, &inputs) - .map_err(|error| CandidateInventoryError::PhysicalImplementation { - group, - candidate: index, - reason: error.to_string(), - })?; - let id = - PhysicalCandidateId(inventory.operator_nodes_of_candidates_for_memo_groups.len()); - inventory.implementation_roots_of_group[group].push(id); - inventory.operator_nodes_of_candidates_for_memo_groups.push( - OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![CandidateForMemoGroupId { - group: MemoGroupId(group as u32), - index, - }], - is_implementation_root: true, - accuracy_budget_parts: candidate.accuracy_budget_parts, - physical_node: implemented, - }, - ); - inventory.producer_nodes_per_input.push(Vec::new()); - } - } - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - let mut boundaries = HashMap::new(); - for &child in &candidate.input_groups { - let pointer = Rc::as_ptr(&memo.groups[child.0 as usize].logical_node); - if boundaries - .insert(pointer, child) - .is_some_and(|previous| previous != child) - { - return Err(invalid("one boundary node names different input groups")); - } - } - if boundaries.contains_key(&Rc::as_ptr(&candidate.root_node)) { - return Err(CandidateInventoryError::Unsupported { - group, - candidate: index, - reason: "root aliases an input group without an operator", - }); - } - let mut builder = ImmediateCandidateBuilder { - inventory: &mut inventory, - compatible_producers_cache: &mut *compatible_producers_cache, - boundaries, - private_nodes: HashMap::new(), - used_boundaries: HashSet::new(), - owner: CandidateForMemoGroupId { - group: MemoGroupId(group as u32), - index, - }, - owner_accuracy_budget_parts: candidate.accuracy_budget_parts, - }; - let root_id = builder.inventory.implementation_roots_of_group[group][index]; - let implemented = Rc::clone( - &builder - .inventory - .operator_nodes_of_candidates_for_memo_groups[root_id.0] - .physical_node, - ); - builder.add(&implemented, Some(root_id))?; - if candidate - .input_groups - .iter() - .any(|group| !builder.used_boundaries.contains(group)) - { - return Err(invalid( - "declared input group has no canonical node boundary", - )); - } - } - } - for group in output_groups { - let expected = &memo.groups[group.0 as usize].logical_node; - let mut compatible_candidates = Vec::new(); - for &id in &inventory.implementation_roots_of_group[group.0 as usize] { - let template = - &inventory.operator_nodes_of_candidates_for_memo_groups[id.0].physical_node; - if compatible_output(expected, template)? { - compatible_candidates.push(id); - } - } - inventory.query_output_nodes.push(compatible_candidates); - } - inventory.validate().map_err(|error| invalid(&error))?; - Ok(inventory) -} - -struct ImmediateCandidateBuilder<'a> { - inventory: &'a mut CandidateInventory, - compatible_producers_cache: &'a mut MemoGroupOutputToCompatibleProducersCache, - boundaries: HashMap<*const OperatorNode, MemoGroupId>, - private_nodes: HashMap<*const OperatorNode, PhysicalCandidateId>, - used_boundaries: HashSet, - /// The memo candidate being lowered; owner of every node it adds. - owner: CandidateForMemoGroupId, - /// The owner's `accuracy_budget_parts`, which its private nodes carry. - owner_accuracy_budget_parts: usize, -} - -impl ImmediateCandidateBuilder<'_> { - fn add( - &mut self, - node: &Rc, - reserved: Option, - ) -> Result { - if let Some(&existing) = self.private_nodes.get(&Rc::as_ptr(node)) { - return Ok(existing); - } - if node.timing.is_some() || node.materialization != Materialization::NotMaterialized { - return Err(CandidateInventoryError::Unsupported { - group: self.owner.group.0 as usize, - candidate: self.owner.index, - reason: "assigned execution timing or materialization", - }); - } - let mut requirements = Vec::new(); - for input in super::reconstruction::ordered_inputs(node) { - let producers = if let Some(&group) = self.boundaries.get(&Rc::as_ptr(&input)) { - self.used_boundaries.insert(group); - let key = MemoGroupOutputKey::new(group, &input) - .map_err(CandidateInventoryError::InvalidMemo)?; - if let Some(compatible) = self.compatible_producers_cache.get(&key) { - compatible - } else { - let mut compatible = Vec::new(); - for &candidate in - &self.inventory.implementation_roots_of_group[group.0 as usize] - { - let template = &self.inventory.operator_nodes_of_candidates_for_memo_groups - [candidate.0] - .physical_node; - if compatible_output(&input, template)? { - compatible.push(candidate); - } - } - self.compatible_producers_cache.remember(key, compatible) - } - } else { - vec![self.add(&input, None)?].into() - }; - requirements.push(producers); - } - let id = if let Some(id) = reserved { - self.inventory.producer_nodes_per_input[id.0] = requirements; - id - } else { - let id = PhysicalCandidateId( - self.inventory - .operator_nodes_of_candidates_for_memo_groups - .len(), - ); - self.inventory.producer_nodes_per_input.push(requirements); - self.inventory - .operator_nodes_of_candidates_for_memo_groups - .push(OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![self.owner], - is_implementation_root: false, - accuracy_budget_parts: self.owner_accuracy_budget_parts, - physical_node: Rc::clone(node), - }); - id - }; - self.private_nodes.insert(Rc::as_ptr(node), id); - Ok(id) - } -} - -fn compatible_output( - expected: &OperatorNode, - producer: &OperatorNode, -) -> Result { - super::reconstruction::output_contract_satisfied(expected, producer) - .map_err(CandidateInventoryError::InvalidMemo) -} diff --git a/crates/physical-asap-optimization/src/catalog/mod.rs b/crates/physical-asap-optimization/src/catalog/mod.rs index eb3069ead..391775557 100644 --- a/crates/physical-asap-optimization/src/catalog/mod.rs +++ b/crates/physical-asap-optimization/src/catalog/mod.rs @@ -1,163 +1,28 @@ -//! Stage 2's physical candidates: implement each memo candidate's -//! operators, enumerate its timing and materialization, record which -//! candidates can produce each input, and prune what cannot be used -//! ([`generate_physical_asap_memo_candidates`]); and realize one selected -//! assignment as a physical plan (#1047). The output type, +//! Stage 2's physical candidates: expand each logical candidate into one +//! physical candidate per valid $(\tau, \ell)$, and each logical edge +//! into the physical edges between the two candidates' alternatives +//! ([`generate_physical_asap_memo_candidates`]). The output type, //! `PhysicalASAPMemoCandidates`, lives in `asap_types::stages`; Stage 3 //! selects from it. -mod configurations; -mod factoring; -mod inventory; -mod pruning; -mod realize; -mod reconstruction; -pub use configurations::ConfigurationError; -pub use inventory::CandidateInventoryError; -pub use realize::Stage2AssignmentRealizer; - -use asap_types::stages::{ - LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateId, ReaderConstraint, -}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -/// Why [`generate_physical_asap_memo_candidates`] failed. -#[derive(Debug, thiserror::Error)] -pub enum ErrorGeneratingPhysicalASAPMemoCandidates { - /// Lowering the memo's candidates to operator nodes failed. - #[error(transparent)] - Inventory(#[from] CandidateInventoryError), - /// Enumerating timings and materializations failed. - #[error(transparent)] - Configuration(#[from] ConfigurationError), - /// Structural pruning found the candidates malformed. - #[error("{0}")] - Pruning(String), -} - -/// Stage 2: implement every operator node of every candidate in `logical`'s -/// memo, enumerate the timings and materializations the data and deployment -/// allow (`query_time_results_retained`; no query-time raw scans unless -/// `raw_data_retained`; no exact per-entity aggregate the executor cannot -/// run), record which candidates may produce each input, and prune what -/// cannot be used. Every configuration not generated and every candidate -/// pruned is recorded in the output's `pruned_candidates` with its -/// `PruneReason`. Prices nothing. +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::stages::{LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates}; +use asap_types::workload::DataWorkload; + +/// Stage 2: expand every logical candidate of `logical` that survived Pass 2 +/// into its physical alternatives (decision-tree levels 3–4: timing and +/// lifetime) the data and the deployment's `capabilities` allow, and every +/// logical edge into its physical edges with $\alpha^{\log}$ and +/// $\alpha^{\mathrm{phys}}$. Every alternative rejected is recorded in the +/// output's `pruned_candidates` with its `PruneReason`; a value outside a +/// dimension's declared value set (`AcrossEvaluations` on a candidate of a +/// group with no known grid) is not offered and not recorded. Costs nothing, and +/// cannot fail. pub fn generate_physical_asap_memo_candidates( logical: &LogicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], data: &DataWorkload, - query_time_results_retained: bool, - raw_data_retained: bool, -) -> Result { - let memo = &logical.memo; - let output_groups: Vec<_> = asap_types::stages::output_references(&memo.query_roots) - .into_iter() - .map(|(_, group)| group) - .collect(); - let inventory = inventory::build_candidate_inventory(memo, &output_groups)?; - let configured = configurations::enumerate_candidate_configurations( - &inventory, - &memo.query_roots, - entries, - data, - query_time_results_retained, - raw_data_retained, - )?; - pruning::prune(configured).map_err(ErrorGeneratingPhysicalASAPMemoCandidates::Pruning) -} - -/// Each output reference's candidates ([`PhysicalASAPMemoCandidates::query_output_candidates`]). -fn query_output_candidates(physical: &PhysicalASAPMemoCandidates) -> Vec> { - (0..physical.output_reference_count()) - .map(|output| physical.query_output_candidates(output).collect()) - .collect() -} - -/// `ids` name existing candidates (of `count`), each once. -fn validate_ids(ids: &[PhysicalCandidateId], count: usize, context: &str) -> Result<(), String> { - let mut seen = std::collections::HashSet::new(); - for id in ids { - if id.0 >= count { - return Err(format!("{context} references missing candidate {}", id.0)); - } - if !seen.insert(*id) { - return Err(format!("{context} repeats candidate {}", id.0)); - } - } - Ok(()) -} - -/// Every producer, operator node and reader constraint of `physical` names an -/// existing candidate or output reference, and each query reader constraint -/// partitions the output references. -fn validate_candidates(physical: &PhysicalASAPMemoCandidates) -> Result<(), String> { - let count = physical.candidates.len(); - let outputs_count = physical.output_reference_count(); - let invalid = |message: &str| format!("invalid physical ASAP memo candidates: {message}"); - for constraint in &physical.reader_constraints { - let outputs = match constraint { - ReaderConstraint::MinimumQueryReaders { query_outputs, .. } - | ReaderConstraint::HighestSharedQueryMaterialization { query_outputs, .. } => { - let outputs: Vec<_> = query_outputs.iter().flatten().copied().collect(); - let unique: std::collections::HashSet<_> = outputs.iter().copied().collect(); - if outputs.len() != outputs_count || unique.len() != outputs.len() { - return Err(invalid( - "query reader buckets must partition every output exactly once", - )); - } - outputs - } - ReaderConstraint::AllowedOutputs { outputs, .. } => outputs.clone(), - ReaderConstraint::IncompatibleOutputs { left, right, .. } => vec![*left, *right], - }; - if constraint.candidate().0 >= count - || outputs.iter().any(|&output| output >= outputs_count) - { - return Err(invalid( - "reader constraint references missing candidate or output", - )); - } - } - for (index, candidate) in physical.candidates.iter().enumerate() { - if candidate.operator_node.0 >= physical.operator_nodes_of_candidates_for_memo_groups.len() - { - return Err(invalid(&format!( - "candidate {index} references missing operator node" - ))); - } - for (input, producer_candidates) in - candidate.producer_candidates_per_input.iter().enumerate() - { - validate_ids( - producer_candidates, - count, - &format!("candidate {index}, requirement {input}"), - ) - .map_err(|message| invalid(&message))?; - } - } - Ok(()) -} - -/// Catalog internals that plan selection's unit tests exercise together with -/// its solver. -#[cfg(feature = "test-support")] -pub mod test_support { - pub use super::configurations::{ - compatible_timing, enumerate_candidate_configurations, - may_keep_across_evaluations_per_sole_reader_query, - }; - pub use super::factoring::{ - factor_terminal_readers, MemoGroupOutputKey, MemoGroupOutputToCompatibleProducersCache, - }; - pub use super::inventory::{ - build_candidate_inventory, build_with_compatible_producers_cache, CandidateInventory, - }; - pub use super::pruning::usable_candidates; - pub use super::realize::{materialize_selected_candidates, restore_query_roots}; - pub use super::reconstruction::{ - ordered_inputs, output_contract_satisfied, paired_node_identities, - reconstruct_selected_candidates, ReconstructedCandidates, - }; + capabilities: &DeploymentCapabilities, +) -> PhysicalASAPMemoCandidates { + let _ = (logical, data, capabilities); + todo!("#1207 PR 5: physical expansion and physical edges") } diff --git a/crates/physical-asap-optimization/src/catalog/pruning.rs b/crates/physical-asap-optimization/src/catalog/pruning.rs deleted file mode 100644 index 4cde4cb9e..000000000 --- a/crates/physical-asap-optimization/src/catalog/pruning.rs +++ /dev/null @@ -1,195 +0,0 @@ -//! Remove provably unusable configurations before constructing solver variables. -//! No costs or dominance assumptions are involved: finite input realizability, -//! potential output reach and necessary reader counts are sufficient proofs. -use asap_types::stages::{ - PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalCandidateKey, PruneReason, - PrunedCandidate, ReaderConstraint, -}; -use asap_types::stages::{ProducerCandidates, ProducerSetInterner}; -use std::collections::HashMap; - -/// `configured` without the candidates `prune_reasons` rules out, its -/// IDs renumbered densely in their order; each removed one is appended to -/// its `pruned_candidates` with the reason. `configured` is already -/// validated: configuration enumeration ends by validating its output. -pub fn prune(configured: PhysicalASAPMemoCandidates) -> Result { - let count = configured.candidates.len(); - let reasons = prune_reasons(&configured); - let keep: Vec = reasons.iter().map(Option::is_none).collect(); - let mut pruned_candidates = configured.pruned_candidates.clone(); - for (candidate, reason) in configured.candidates.iter().zip(&reasons) { - if let Some(reason) = *reason { - pruned_candidates.push(PrunedCandidate { - group: configured.operator_nodes_of_candidates_for_memo_groups - [candidate.operator_node.0] - .group(), - candidate: PhysicalCandidateKey { - operator_node: candidate.operator_node, - timing: candidate.timing, - materialization: candidate.materialization, - }, - reason, - }); - } - } - let mut mapping = vec![None; count]; - let mut next = 0; - for (id, retained) in keep.iter().enumerate() { - if *retained { - mapping[id] = Some(PhysicalCandidateId(next)); - next += 1; - } - } - let mut producer_sets = ProducerSetInterner::default(); - let mut remapped: HashMap = HashMap::new(); - let candidates = configured - .candidates - .into_iter() - .enumerate() - .filter_map(|(id, mut candidate)| { - if !keep[id] { - return None; - } - for input in &mut candidate.producer_candidates_per_input { - *input = remapped - .entry(input.clone()) - .or_insert_with(|| { - producer_sets.intern( - input - .iter() - .filter_map(|id| mapping[id.0]) - .collect::>(), - ) - }) - .clone(); - } - Some(candidate) - }) - .collect(); - let reader_constraints = configured - .reader_constraints - .into_iter() - .filter_map(|mut constraint| { - let id = constraint.candidate_mut(); - *id = mapping[id.0]?; - Some(constraint) - }) - .collect(); - // Operator nodes are kept whole: `operator_node` indexes them unchanged. - let mut pruned = PhysicalASAPMemoCandidates::new( - configured.operator_nodes_of_candidates_for_memo_groups, - candidates, - configured.query_roots, - reader_constraints, - ); - pruned.pruned_candidates = pruned_candidates; - super::validate_candidates(&pruned)?; - Ok(pruned) -} - -/// Which candidates of `physical` [`prune`] keeps. -#[cfg(feature = "test-support")] -pub fn usable_candidates(physical: &PhysicalASAPMemoCandidates) -> Vec { - prune_reasons(physical) - .iter() - .map(Option::is_none) - .collect() -} - -/// Why each candidate of `physical` cannot be used, `None` for a usable -/// one: the reason of the first round that ruled it out, and within a -/// round the first of: an input with no buildable producer, no query output -/// reached, too few query readers. -fn prune_reasons(physical: &PhysicalASAPMemoCandidates) -> Vec> { - let count = physical.candidates.len(); - let query_output_candidates = super::query_output_candidates(physical); - let mut reasons: Vec> = vec![None; count]; - loop { - let enabled: Vec = reasons.iter().map(Option::is_none).collect(); - // Least fixed point: a cycle without a finite implementation is not a - // producer. A cycle with an independent escape candidate remains usable. - let mut realizable = vec![false; count]; - loop { - let mut changed = false; - for (id, candidate) in physical.candidates.iter().enumerate() { - if enabled[id] - && !realizable[id] - && candidate - .producer_candidates_per_input - .iter() - .all(|input| input.iter().any(|id| realizable[id.0])) - { - realizable[id] = true; - changed = true; - } - } - if !changed { - break; - } - } - let mut reached = vec![vec![false; query_output_candidates.len()]; count]; - for (output, root) in query_output_candidates.iter().enumerate() { - let mut pending = root.clone(); - while let Some(id) = pending.pop() { - if realizable[id.0] && !reached[id.0][output] { - reached[id.0][output] = true; - pending.extend( - physical.candidates[id.0] - .producer_candidates_per_input - .iter() - .flat_map(|input| input.iter().copied()), - ); - } - } - } - // Intersect all allowed-reader restrictions before counting potential - // distinct queries. Potential reach is an upper bound, never a demand. - for constraint in &physical.reader_constraints { - if let ReaderConstraint::AllowedOutputs { candidate, outputs } = constraint { - for (output, value) in reached[candidate.0].iter_mut().enumerate() { - *value &= outputs.contains(&output); - } - } - } - let mut round: Vec> = realizable - .iter() - .zip(&reached) - .zip(&enabled) - .map(|((realizable, outputs), enabled)| { - if !enabled || *realizable && outputs.iter().any(|value| *value) { - None - } else if !realizable { - Some(PruneReason::InputHasNoBuildableProducer) - } else { - Some(PruneReason::ReachesNoQueryOutput) - } - }) - .collect(); - for constraint in &physical.reader_constraints { - if let ReaderConstraint::MinimumQueryReaders { - candidate, - query_outputs, - minimum, - } = constraint - { - let queries = query_outputs - .iter() - .filter(|outputs| outputs.iter().any(|output| reached[candidate.0][*output])) - .count(); - if queries < *minimum { - round[candidate.0].get_or_insert(PruneReason::TooFewQueryReaders); - } - } - } - let mut changed = false; - for (reason, round) in reasons.iter_mut().zip(round) { - if reason.is_none() && round.is_some() { - *reason = round; - changed = true; - } - } - if !changed { - return reasons; - } - } -} diff --git a/crates/physical-asap-optimization/src/catalog/realize.rs b/crates/physical-asap-optimization/src/catalog/realize.rs deleted file mode 100644 index f473709c1..000000000 --- a/crates/physical-asap-optimization/src/catalog/realize.rs +++ /dev/null @@ -1,325 +0,0 @@ -//! Apply an explicit selected materialization assignment through real Stage 2. -//! -//! No fallback modes are chosen. Physical rewrites or timing splits that change -//! candidate identity are rejected until represented by physical enumeration. -//! This validates execution legality, not cost estimates or accuracy targets. -use std::collections::HashMap; -use std::rc::Rc; - -use crate::optimize_physical_asap_chosen; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::physical::Materialization; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use asap_types::stages::{ - AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, - PhysicalCandidateError, RealizedPlan, -}; - -use super::reconstruction::{ - output_contract_satisfied, paired_node_identities, reconstruct_selected_candidates, -}; - -/// Realize `selected` with each selected node at its candidate's -/// materialization. Output -/// references must follow `map_operator_refs` traversal of the original -/// roots in workload order (one reference for an operator root). Repeated -/// scalar references remain separate positions; literal-only scalar roots -/// use none. -pub fn materialize_selected_candidates( - physical_candidates: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, - original_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result { - if entries.len() != original_roots.len() { - return Err("workload entries and query roots differ in length".into()); - } - let rebuilt = reconstruct_selected_candidates(physical_candidates, selected)?; - // `reconstruct_selected_candidates` checked every selected candidate exists. - let materializations: Vec<_> = selected - .candidate_of_node - .iter() - .map(|id| physical_candidates.candidates[id.0].materialization) - .collect(); - let mut node_by_pointer = HashMap::new(); - for (node, operator_node) in rebuilt.operator_node_of_selected_node.iter().enumerate() { - node_by_pointer.insert(Rc::as_ptr(operator_node), node); - } - let roots = restore_query_roots(original_roots, &rebuilt.outputs)?; - let mut assignment_error = None; - let mut seen = vec![false; materializations.len()]; - let physical = optimize_physical_asap_chosen( - "selected-candidate-connections", - &roots, - entries, - data, - query_time_results_retained, - &mut |logical, _, options| { - let node = node_by_pointer.get(&logical).copied(); - let requested = if let Some(node) = node { - seen[node] = true; - materializations[node] - } else { - Materialization::NotMaterialized - }; - if !options.contains(&requested) { - let message = format!( - "selected materialization {requested:?} is not offered for selected node {node:?}" - ); - assignment_error = Some(if node.is_some() { - PhysicalCandidateError::RejectedAssignment(message) - } else { - PhysicalCandidateError::InvalidOrUnsupported(message) - }); - return None; - } - Some(requested) - }, - ) - .map_err(|error| error.to_string())?; - if let Some(error) = assignment_error { - return Err(error); - } - let physical = physical.ok_or_else(|| { - PhysicalCandidateError::RejectedAssignment( - "selected materialization assignment violates Stage 2 timing or dependency rules" - .into(), - ) - })?; - physical.dag.validate().map_err(|error| error.to_string())?; - let physical_outputs = ordered_output_references(&physical.roots); - let identities = paired_node_identities(&rebuilt.outputs, &physical_outputs)?; - for (node, mode) in materializations.iter().enumerate() { - if *mode != Materialization::NotMaterialized && !seen[node] { - // The decoder already established node identity. Stage 2 did - // not offer this selected node (e.g. a lower shared non-summary), - // so the assignment is infeasible rather than an adapter failure. - return Err(PhysicalCandidateError::RejectedAssignment(format!( - "selected node {node} is not an eligible materialization node in the selected graph" - ))); - } - } - let operator_node_of_selected_node = rebuilt - .operator_node_of_selected_node - .into_iter() - .enumerate() - .map(|(node, operator_node)| { - let physical = identities - .get(&Rc::as_ptr(&operator_node)) - .cloned() - .ok_or_else(|| format!("selected node {node} has no physical identity"))?; - if physical.materialization != materializations[node] { - return Err(format!( - "selected node {node} materialization changed during physical reconstruction" - )); - } - Ok(physical) - }) - .collect::, String>>()?; - let range_evaluations: Vec<_> = entries.iter().map(|entry| entry.range_evaluation).collect(); - let tumbling_window_retention_ms = - asap_cost::tumbling_window_retention_ms_of_plan(&physical.dag, &range_evaluations); - Ok(RealizedPlan { - plan: physical, - tumbling_window_retention_ms, - operator_node_of_selected_node, - }) -} - -/// Stage 2's [`AssignmentRealizer`], over the planning context: the -/// workload's original query roots (one per entry of `entries`), its data, -/// and whether the deployment keeps query-time results across -/// evaluations. The facade builds it, passes it to Stage 3, and builds the -/// selected plan with it. -#[derive(Debug, Clone, Copy)] -pub struct Stage2AssignmentRealizer<'a> { - pub original_roots: &'a [QueryRoot], - pub entries: &'a [QueryWorkloadEntry], - pub data: &'a DataWorkload, - pub query_time_results_retained: bool, -} - -impl AssignmentRealizer for Stage2AssignmentRealizer<'_> { - /// `materialize_selected_candidates`; each node also keeps its - /// candidate's timing. - fn realize( - &self, - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, - ) -> Result { - let realized = materialize_selected_candidates( - physical, - selected, - self.original_roots, - self.entries, - self.data, - self.query_time_results_retained, - )?; - for (id, node) in selected - .candidate_of_node - .iter() - .zip(&realized.operator_node_of_selected_node) - { - let timing = physical.candidates[id.0].timing; - if node.timing != Some(timing) { - return Err(PhysicalCandidateError::InvalidOrUnsupported(format!( - "candidate {} requested timing {timing:?}, Stage 2 produced {:?}", - id.0, node.timing - ))); - } - } - Ok(realized) - } -} - -fn ordered_output_references(roots: &[QueryRoot]) -> Vec> { - asap_types::stages::output_references(roots) - .into_iter() - .map(|(_, node)| node) - .collect() -} - -pub fn restore_query_roots( - original: &[QueryRoot], - outputs: &[Rc], -) -> Result, String> { - let references = ordered_output_references(original); - if references.len() != outputs.len() { - return Err("output references do not match original query-root mapping".into()); - } - for (reference, output) in references.iter().zip(outputs) { - if !output_contract_satisfied(reference, output)? { - return Err(format!( - "selected output reference has incompatible result or schema: expected {:?} {:?}, received {:?} {:?}", - reference.result_kind, reference.schema, output.result_kind, output.schema - )); - } - } - let mut next = outputs.iter(); - Ok(original - .iter() - .map(|root| match root { - QueryRoot::Operator(_) => QueryRoot::Operator(Rc::clone(next.next().unwrap())), - QueryRoot::Scalar(expression) => QueryRoot::Scalar( - expression.map_operator_refs(&mut |_| Rc::clone(next.next().unwrap())), - ), - }) - .collect()) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - use asap_logical_asap_optimization::test_support::try_populate_pass1; - use asap_logical_asap_optimization::test_support::{ - entry, lower_promql, sharing_rules_and_shared_state_groups, - }; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::stages::{ - LogicalASAPMemoCandidates, PhysicalCandidateId, SelectedEdge, SelectedNodeId, - }; - use asap_types::workload::QueryRecurrence; - - /// Add a node running `candidate` and, for each input, a node of its - /// first unmaterialized producer (else its first producer), - /// recursively; returns the new node. - fn select_from( - physical: &PhysicalASAPMemoCandidates, - candidate: PhysicalCandidateId, - selected: &mut DagOfSelectedCandidates, - ) -> SelectedNodeId { - let node = SelectedNodeId(selected.candidate_of_node.len()); - selected.candidate_of_node.push(candidate); - for (input, producer_candidates) in physical.candidates[candidate.0] - .producer_candidates_per_input - .iter() - .enumerate() - { - let producer = producer_candidates - .iter() - .copied() - .find(|s| { - physical.candidates[s.0].materialization == Materialization::NotMaterialized - }) - .unwrap_or(producer_candidates[0]); - let producer = select_from(physical, producer, selected); - selected.edges.push(SelectedEdge { - producer, - consumer: node, - input, - }); - } - node - } - - // The trait object returns what the catalog function returns with each - // candidate's own materialization. - #[test] - fn stage2_realizer_matches_direct_catalog_calls() { - let root = QueryRoot::Operator(lower_promql( - "sum by (job) (sum_over_time(m[1m]))", - AccuracyTarget::exact(), - )); - let roots = vec![root.clone()]; - let initial = generate_dag_of_logical_memo_groups(&[root]); - let per_computation = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - roots[0].clone() - ])) - .unwrap(); - let memo = try_populate_pass1(initial, &per_computation).unwrap(); - let logical = LogicalASAPMemoCandidates { - memo: sharing_rules_and_shared_state_groups(memo).unwrap(), - }; - let entries = vec![entry(QueryRecurrence::Unknown)]; - let data = DataWorkload::default(); - let physical = crate::catalog::generate_physical_asap_memo_candidates( - &logical, &entries, &data, false, true, - ) - .unwrap(); - let selected = physical - .query_output_candidates(0) - .map(|candidate| { - let mut selected = DagOfSelectedCandidates { - candidate_of_node: Vec::new(), - edges: Vec::new(), - node_of_output_reference: Vec::new(), - }; - let root = select_from(&physical, candidate, &mut selected); - selected.node_of_output_reference.push(root); - selected - }) - .find(|selected| reconstruct_selected_candidates(&physical, selected).is_ok()) - .expect("some selection reconstructs"); - assert!(selected.candidate_of_node.len() > 1); - let realizer = Stage2AssignmentRealizer { - original_roots: &roots, - entries: &entries, - data: &data, - query_time_results_retained: false, - }; - let realizer: &dyn AssignmentRealizer = &realizer; - - let direct = - materialize_selected_candidates(&physical, &selected, &roots, &entries, &data, false) - .unwrap(); - let through_trait = realizer.realize(&physical, &selected).unwrap(); - assert_eq!( - format!("{:?}", direct.plan), - format!("{:?}", through_trait.plan) - ); - assert_eq!( - direct.operator_node_of_selected_node, - through_trait.operator_node_of_selected_node - ); - assert_eq!( - direct.tumbling_window_retention_ms, - through_trait.tumbling_window_retention_ms - ); - } -} diff --git a/crates/physical-asap-optimization/src/catalog/reconstruction.rs b/crates/physical-asap-optimization/src/catalog/reconstruction.rs deleted file mode 100644 index fd7f0068b..000000000 --- a/crates/physical-asap-optimization/src/catalog/reconstruction.rs +++ /dev/null @@ -1,231 +0,0 @@ -//! Reconstruct selected immediate operators, preserving candidate identity. -//! -//! This decoder checks typed connections, not Stage 2 execution legality or -//! query accuracy targets. The caller must run Stage 2 and the plan validator. -use std::collections::HashMap; -use std::rc::Rc; - -use asap_accuracy::propagation::recompose_guarantees; -use asap_types::ir::{OperatorNode, QueryRoot}; - -use asap_types::stages::{ - DagOfSelectedCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateId, -}; - -/// Enumerate in exactly the traversal order used when rebinding. `children()` -/// puts direct children before scalar references; `map_children` follows fields. -pub fn ordered_inputs(node: &OperatorNode) -> Vec> { - let mut inputs = Vec::new(); - node.operator.map_children(|child| { - inputs.push(Rc::clone(child)); - Rc::clone(child) - }); - inputs -} - -/// Open schemas do not promise completeness; an equivalent derived output may -/// strengthen that promise to closed. Supplied uniqueness must imply every -/// required key; additional proven keys are allowed. Every field, configuration -/// parameter and time axis remains exact. Serialization includes sketch error/failure -/// metadata deliberately omitted by some parameter PartialEq implementations. -pub fn output_contract_satisfied( - expected: &OperatorNode, - output: &OperatorNode, -) -> Result { - let mut supplied = output.schema.clone(); - if !expected.schema.closed { - supplied.closed = false; - } - if !expected.schema.unique_keys.iter().all(|required| { - supplied - .unique_keys - .iter() - .any(|provided| provided.iter().all(|column| required.contains(column))) - }) { - return Ok(false); - } - supplied.unique_keys = expected.schema.unique_keys.clone(); - Ok(expected.result_kind == output.result_kind - && expected.schema == supplied - && serde_json::to_string(&expected.schema).map_err(|error| error.to_string())? - == serde_json::to_string(&supplied).map_err(|error| error.to_string())?) -} - -pub struct ReconstructedCandidates { - pub outputs: Vec>, - /// The operator node each selected node became, indexed by - /// [`SelectedNodeId`](asap_types::stages::SelectedNodeId), with - /// guarantees recomputed; each a distinct operator node. - pub(crate) operator_node_of_selected_node: Vec>, -} - -/// The nodes of `selected` rebuilt over their selected producers, one -/// operator node each, with guarantees recomposed. Checks the selection's -/// structure, not its cost. -pub fn reconstruct_selected_candidates( - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, -) -> Result { - super::validate_candidates(physical)?; - selected.validate(physical)?; - let count = selected.candidate_of_node.len(); - let mut inputs = selected - .candidate_of_node - .iter() - .map(|candidate| { - vec![ - None; - physical.candidates[candidate.0] - .producer_candidates_per_input - .len() - ] - }) - .collect::>(); - let mut consumers = vec![Vec::new(); count]; - for edge in &selected.edges { - inputs[edge.consumer.0][edge.input] = Some(edge.producer.0); - consumers[edge.producer.0].push(edge.consumer.0); - } - let mut remaining = inputs.iter().map(Vec::len).collect::>(); - let mut ready = Vec::new(); - for (node, candidate) in selected.candidate_of_node.iter().enumerate() { - let template = physical.physical_node(*candidate); - if template.children().len() != inputs[node].len() { - return Err(format!( - "candidate {} template inputs do not match producer requirements", - candidate.0 - )); - } - if remaining[node] == 0 { - ready.push(node); - } - } - let mut rebuilt: Vec>> = vec![None; count]; - while let Some(node) = ready.pop() { - let id = selected.candidate_of_node[node].0; - let template = physical.physical_node(PhysicalCandidateId(id)); - let expected = ordered_inputs(template); - let mut producers = Vec::with_capacity(inputs[node].len()); - for (input, producer) in inputs[node].iter().enumerate() { - let producer = rebuilt[producer.unwrap()].as_ref().unwrap(); - if !output_contract_satisfied(&expected[input], producer)? { - return Err(format!( - "candidate {id} input {input} has incompatible schema or state configuration" - )); - } - producers.push(Rc::clone(producer)); - } - let mut next = 0; - // Re-derive structural metadata over the selected inputs, preserving - // only explicit naming overrides. Logical rewrite templates can retain - // pre-rewrite keys/nullability; copying their entire schema is invalid. - let rebuilt_node = Rc::new( - template - .with_new_children(|_| { - let producer = Rc::clone(&producers[next]); - next += 1; - producer - }) - .map_err(|e| format!("candidate {id}: {e}"))? - .with_guarantee(template.guarantee.clone()), - ); - rebuilt_node - .validate_structure() - .map_err(|e| format!("candidate {id}: {e}"))?; - rebuilt[node] = Some(rebuilt_node); - for &consumer in &consumers[node] { - remaining[consumer] -= 1; - if remaining[consumer] == 0 { - ready.push(consumer); - } - } - } - let rebuilt = rebuilt - .into_iter() - .map(|node| node.ok_or_else(|| "selected nodes form a cycle".to_string())) - .collect::, _>>()?; - let roots = selected - .node_of_output_reference - .iter() - .map(|node| QueryRoot::Operator(Rc::clone(&rebuilt[node.0]))) - .collect::>(); - let outputs = recompose_guarantees(&roots) - .map_err(|e| e.to_string())? - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => Ok(node), - QueryRoot::Scalar(_) => Err("unexpected scalar during operator reconstruction".into()), - }) - .collect::, String>>()?; - let old_outputs = roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => Rc::clone(node), - QueryRoot::Scalar(_) => unreachable!("operator references were constructed above"), - }) - .collect::>(); - let identities = paired_node_identities(&old_outputs, &outputs)?; - let operator_node_of_selected_node = rebuilt - .into_iter() - .map(|node| { - identities - .get(&Rc::as_ptr(&node)) - .cloned() - .ok_or_else(|| "selected node lost during guarantee recomposition".to_string()) - }) - .collect::, _>>()?; - Ok(ReconstructedCandidates { - outputs, - operator_node_of_selected_node, - }) -} - -/// Match corresponding graph positions without structural interning. Reject -/// splitting or merging identities, and rewrites that change immediate operators. -pub fn paired_node_identities( - before: &[Rc], - after: &[Rc], -) -> Result>, String> { - if before.len() != after.len() { - return Err("output reference count changed".into()); - } - let mut identities = HashMap::<*const OperatorNode, Rc>::new(); - let mut reverse = HashMap::new(); - let mut pending = before - .iter() - .cloned() - .zip(after.iter().cloned()) - .collect::>(); - while let Some((old, new)) = pending.pop() { - if let Some(existing) = identities.get(&Rc::as_ptr(&old)) { - if !Rc::ptr_eq(existing, &new) { - return Err("selected node was split into multiple physical nodes".into()); - } - continue; - } - if reverse - .insert(Rc::as_ptr(&new), Rc::as_ptr(&old)) - .is_some_and(|previous| previous != Rc::as_ptr(&old)) - { - return Err("distinct selected nodes were merged".into()); - } - let old_operator = old.operator.map_children(|_| ()); - let new_operator = new.operator.map_children(|_| ()); - // Compare all serialized parameters, including sketch probability and - // non-finite scalar values. PartialEq rejects an unchanged NaN, while - // JSON erases the distinction between NaN and either infinity. - if rmp_serde::to_vec(&old_operator).map_err(|e| e.to_string())? - != rmp_serde::to_vec(&new_operator).map_err(|e| e.to_string())? - { - return Err("operator implementation changed; explicit physical candidate enumeration is required".into()); - } - let old_children = ordered_inputs(&old); - let new_children = ordered_inputs(&new); - if old_children.len() != new_children.len() { - return Err("candidate input count changed".into()); - } - pending.extend(old_children.into_iter().zip(new_children)); - identities.insert(Rc::as_ptr(&old), new); - } - Ok(identities) -} diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/physical-asap-optimization/src/lib.rs index aed2e101c..5ef0d90d1 100644 --- a/crates/physical-asap-optimization/src/lib.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -1,2437 +1,15 @@ -//! #509 Stage 2, physical ASAP-aware optimization: materialization and -//! physical operator implementation. +//! #509 Stage 2, physical ASAP-aware optimization (concepts doc, +//! section 3, "Stage 2: physical alternatives"). //! //! The planner facade calls [`generate_physical_asap_memo_candidates`] on -//! Pass 2's memo (`asap_types::stages::LogicalASAPMemoCandidates`): every -//! operator node of every memo candidate with each allowed timing and -//! materialization, the producers of each input and the pruned candidates +//! Pass 2's output (`asap_types::stages::LogicalASAPMemoCandidates`): each +//! logical candidate expanded into one physical candidate per valid +//! $(\tau, \ell)$, each logical edge into its physical edges with +//! $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$, and the pruned alternatives //! (`asap_types::stages::PhysicalASAPMemoCandidates`). Stage 3 selects from -//! that output, and [`Stage2AssignmentRealizer`] builds the selected -//! assignment as a physical plan. The rules below apply to both, and to the -//! whole-workload reference pricer `optimize_physical_asap` (feature -//! `reference-pricer`, for the developer tools), which turns one -//! CandidateLogicalASAPDAG into its CandidatePhysicalASAPDAGs. The crate -//! depends on no other stage crate. -//! -//! ## Materialization -//! -//! Each summary build (a `SummaryAgg` or a `SummaryTumblingWindowChain`) is one -//! materialization candidate (`MaterializationCandidate`) whose -//! output materialization may store (#509 §Materialization, Algorithm 3). -//! So is the highest node of each other sub-DAG that two or more queries -//! read (e.g. a `Scan` that a 1 m and a 5 m window read): one a root or a -//! node only one query reads consumes. Such a node has only -//! `NotMaterialized` and `MatAtQuery(Memory)`, so a shared sub-DAG adds one -//! option, not one per node. A summary build has up to four options: -//! -//! * `NotMaterialized`: rebuilt at query time by every query that reads it; -//! * `MatAtIngestion(Memory)`: maintained as data arrives and kept for the -//! queries, when the build is eligible (`IngestionPruneReason`); -//! * `MatAtQuery(Memory)`: built once at query time for every query of one -//! evaluation that reads it and reused only within that evaluation, -//! offered for a build two or more queries read; -//! * `MatAtQueryKeptAcrossEvaluations(Memory)`: built at query time and -//! kept across evaluations (#509 Example 4's B3), offered when the -//! deployment keeps query-time results (the caller's -//! `query_time_results_retained`) for a -//! tumbling-window chain that builds one new tumbling window per evaluation -//! (`builds_one_tumbling_window_per_evaluation`): each evaluation builds the -//! newest tumbling window and reuses the others. Both query-time options are offered -//! when both apply, and Stage 3 prices each. -//! -//! A build at ingestion time needs every build below it at ingestion time too -//! (#509 Algorithm 3, line 13), and every shared non-summary node below it -//! `NotMaterialized`; other assignments are not generated. Every -//! other combination of options is one physical candidate, the -//! all-`NotMaterialized` one first: nothing is capped. A node that one -//! query reads at ingestion time and another at query time is copied, one -//! copy per timing (`split_shared_by_timing`). -//! -//! A build may run at ingestion time (S4) only when: -//! -//! * the data workload is continuously ingesting or mixed (unknown arrival -//! counts as not ingesting); -//! * every query reading it repeats and is predictable; -//! * it has no filter: an ingestion-time build reads raw data samples, which -//! carry none of the columns a filter reads; -//! * if it is a counter state (`Rate`, `Increase`, `IRate`), whose update -//! needs each series' samples in time order, the data is continuously -//! ingesting and every source below it is a time series. Assumption: -//! continuously ingesting PromQL data arrives in time order per series, -//! as Prometheus assumes (it rejects out-of-order samples). Table (SQL) -//! sources and mixed data, whose at-rest part may load in any order, are -//! not taken to be ordered; -//! * it is a tumbling-window chain whose readers all evaluate on its tumbling window -//! boundaries, anchored at the epoch (each fixed interval a multiple of the -//! tumbling-window width, and each known phase on that grid), or a whole-window build whose readers -//! all run at one fixed interval with agreeing declared phases, over a -//! window that is not shifted and whose evaluations all end on the grid -//! of its width anchored at the epoch (the interval and a declared phase -//! are multiples of the window; a reader with no declared phase is taken -//! to evaluate at multiples of its interval, as a chain's is), and none -//! is a range evaluation of more than one step per request (one window -//! per request cannot serve every step). -//! So a window equal to its interval (`m[5m]` every 5 minutes) is one -//! tumbling window maintained at ingestion time; -//! * everything below it can run at ingestion time. -//! -//! ## Range evaluations -//! -//! A query whose workload entry declares a -//! [`RangeEvaluation`](asap_types::workload::RangeEvaluation) evaluates at -//! every `step_ms` from `end − span_ms` forward to each request's end. -//! Every step reads the -//! tumbling windows of a tumbling-window chain, so each tumbling-window width must divide the step -//! of every range-evaluated reader. Stage 1 offers only such widths -//! (`QueryWorkloadEntry::evaluation_grid_ms`); a candidate with a chain off -//! a reader's step grid fails with -//! [`PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep`] and the caller -//! prunes it with that reason. -//! -//! ## Physical operator implementation -//! -//! The runtime has one implementation per logical operator except exact -//! top-k, which it cannot run as `Aggregate{[TopK]}`. Stage 2 records that -//! choice in the DAG by rewriting it to a per-group sort followed by a -//! per-group limit, the shape the PromQL frontend uses for generic `topk`. -//! A `Project` over the limited rows restores the logical `[keys.., topk_k]` -//! columns, so an exact root has the same column names as a summary -//! candidate's and as the frontend declares. Over PromQL rows it also keeps -//! the series identity, after them: `topk` answers whole input series, so -//! each answer carries the chosen series' full label set. PromQL rows without -//! one (e.g. of `sum by (job)`) keep their other string label columns instead, -//! and SQL rows their other group keys (the ranked item, e.g. `host` of -//! `GROUP BY host ORDER BY COUNT(*) DESC LIMIT k`). -//! `topk_k` carries the ranked value's type and nullability (the frontend -//! declares a non-null `Utf8` placeholder); casting the value to `Utf8` would -//! not make it the same answer. -//! An `Aggregate{[TopK]}` over a per-entity reduction, or with other -//! measures, a filter or `HAVING`, has no such rewrite, and its candidate -//! fails with a [`PhysicalASAPOptimizationError`]; the caller prunes that -//! candidate alone, with the error as its reason. -//! A summary needs no rewrite: `SummaryAgg` → `SummaryEstimate` already is -//! build → estimate. Every node keeps the guarantee Stage 1 composed for it. -mod catalog; -mod split_by_timing; - -pub use catalog::{ - generate_physical_asap_memo_candidates, CandidateInventoryError, ConfigurationError, - ErrorGeneratingPhysicalASAPMemoCandidates, Stage2AssignmentRealizer, -}; - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::Source; -use asap_types::ir::operator::{AggIntent, Reduction}; -use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; -use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; -use asap_types::ir::schema::{ExactKind, FieldDataType}; -use asap_types::ir::{ - apply_materialization_timings, is_filtered_build, ASAPOp, MaterializationAssignment, - NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr, - SchemaDerivationError, SortKey, -}; -use asap_types::physical::Materialization; -use asap_types::physical::Medium; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::workload::{ - DataArrival, DataWorkload, Predictability, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, -}; -use split_by_timing::split_shared_by_timing; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum PhysicalASAPOptimizationError { - #[error(transparent)] - Structure(#[from] SchemaDerivationError), - #[error(transparent)] - Timing(#[from] ExecutionDataStateError), - #[error("exact top-k has no Sort + Limit implementation: {0}")] - ExactTopK(String), - /// Bad candidate for the workload: a tumbling-window chain whose tumbling window - /// width does not divide the step of a query that reads it as a range - /// request, so some step would end inside a tumbling window and could not read - /// the tumbling windows. Stage 1 offers only tumbling windows on every reader's step grid; - /// the candidate is pruned with this reason. - #[error( - "tumbling-window width {tumbling_window_width_ms} ms does not divide the {step_ms} ms step of \ - a range evaluation that reads it" - )] - TumblingWindowWidthOffRangeStep { - /// The chain's tumbling-window width. - tumbling_window_width_ms: u64, - /// The reader's range-evaluation step. - step_ms: u64, - }, -} - -/// Why a summary build cannot be materialized at ingestion time (S4). -/// Shared by complete-plan Stage 2 and explicit selected-reader eligibility. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum IngestionPruneReason { - /// The data workload is at rest, or its arrival is unknown. - NotIngesting, - /// A query reading it is not repeated. - NotRepeated, - /// A query reading it is not predictable. - NotPredictable, - /// It has a filter, which only a query-time build can apply. - Filtered, - /// It is a counter state, whose update needs each series' samples in - /// time order, over data not taken to arrive in that order: only - /// continuously ingesting time-series (PromQL) sources are. - CounterNeedsTimeOrder, - /// A whole-window build whose readers do not all run at one fixed - /// interval and one phase, or whose window is shifted or not a range. - WindowNotFixed, - /// A whole-window build whose evaluations do not all end on the grid of - /// its window's width anchored at the epoch, where the backend builds - /// it (e.g. `m[20s]` every 20 s at phase 5 s, every 30 s, or `m[15m]` - /// every 5 minutes, which tumbling windows serve): the interval or the declared - /// phase is not a multiple of the window. - WindowOffEpochGrid, - /// A whole-window build read by a range evaluation of more than one - /// step per request: it keeps one window per request, not one per step. - RangeEvaluatedWholeWindow, - /// A tumbling-window chain read at a known phase off its tumbling-window boundaries, - /// which are anchored at the epoch (e.g. 20 s tumbling windows read at phase 5 s): - /// built once each, its tumbling windows cannot end at every evaluation. - TumblingWindowsOffPhase, - /// A tumbling-window chain read at a fixed interval that its width does - /// not divide (e.g. 15 s tumbling windows read every 10 s): built once - /// each on the grid anchored at the epoch, its tumbling windows cannot - /// end at every evaluation. - TumblingWindowsOffInterval, - /// Something below it can only run at query time. - UpstreamAtQueryTime(String), - /// It is not a summary build (e.g. a shared `Scan`): only a summary's - /// state is maintained as data arrives. - NotASummaryBuild, -} - -/// Workload eligibility for one summary and only its selected query readers. -/// These are local options, not a complete physical assignment. Ingestion still -/// requires selected upstream implementations to pass timing/down-closure checks. -#[derive(Debug)] -pub(crate) struct SummaryMaterializationEligibility { - pub(crate) at_ingestion: Result<(), IngestionPruneReason>, - pub(crate) may_keep_across_evaluations: bool, - /// A tumbling window's selected readers must also meet their range-step grids. - pub(crate) range_compatibility: Result<(), PhysicalASAPOptimizationError>, -} - -#[derive(Debug, thiserror::Error)] -pub(crate) enum SummaryEligibilityError { - #[error("selected reader {query} has no workload entry")] - UnknownReader { query: usize }, - #[error("materialization eligibility requires a summary build or tumbling-window chain")] - NotSummary, - #[error("materialization eligibility requires at least one selected reader")] - NoSelectedReaders, - #[error("tumbling-window width must be positive")] - ZeroPaneWidth, -} - -/// Evaluate an implemented summary against explicit selected query indices. -/// Never supply the union of potential readers. Duplicate indices count once; -/// missing workload entries are rejected as malformed reader references. The node must carry the actual bound source/filter/window subtree, -/// and `data` describes that source's arrival. This function does not implement -/// operators, assign upstream timing, or prove complete execution legality. -pub(crate) fn summary_materialization_eligibility( - node: &OperatorNode, - selected_readers: &[usize], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result { - if !is_summary_build(node) { - return Err(SummaryEligibilityError::NotSummary); - } - if selected_readers.is_empty() { - return Err(SummaryEligibilityError::NoSelectedReaders); - } - if matches!( - node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 0, - .. - }) - ) { - return Err(SummaryEligibilityError::ZeroPaneWidth); - } - if let Some(&query) = selected_readers - .iter() - .find(|&&query| query >= entries.len()) - { - return Err(SummaryEligibilityError::UnknownReader { query }); - } - let mut readers = selected_readers.to_vec(); - readers.sort_unstable(); - readers.dedup(); - Ok(summary_reader_eligibility( - node, - &readers, - entries, - data, - query_time_results_retained, - )) -} - -fn summary_reader_eligibility( - node: &OperatorNode, - readers: &[usize], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> SummaryMaterializationEligibility { - let recurrences: Option> = readers - .iter() - .map(|&query| entries.get(query).map(|entry| &entry.recurrence)) - .collect(); - let may_keep_across_evaluations = query_time_results_retained - && match (node.asap(), recurrences) { - ( - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }), - Some(recurrences), - ) => builds_one_tumbling_window_per_evaluation(*tumbling_window_width_ms, recurrences), - _ => false, - }; - SummaryMaterializationEligibility { - at_ingestion: ingestion_eligibility(node, readers, entries, data), - may_keep_across_evaluations, - range_compatibility: check_summary_range_steps(node, readers, entries), - } -} - -/// One summary build, or one highest shared non-summary node, Stage 2 may -/// store, with its materialization options. -/// It is not a whole-workload plan like [`PhysicalWorkloadPlan`]: it is Stage -/// 2's enumeration bookkeeping, and the physical candidates are the -/// combinations of the materialization candidates' options. For example, -/// over one metric: -/// -/// ```text -/// q1: quantile_over_time(0.99, latency[5m]) every 1m -/// q2: max_over_time(latency[5m]) every 1m -/// ``` -/// -/// If Stage 1 chose KLL tumbling windows for q1 and exact max tumbling windows for q2, Stage 2 -/// finds two materialization candidates: -/// -/// | materialization candidate | readers | options | -/// |----------------------------------|---------|---------------------------------| -/// | Kll ×5 tumbling windows (q1) | q1 | NotMaterialized, MatAtIngestion | -/// | Max acc ×5 tumbling windows (q2) | q2 | NotMaterialized, MatAtIngestion | -/// -/// One read by two or more queries also gets `MatAtQuery` (built once per -/// evaluation and reused), and so does the highest node of a shared -/// non-summary sub-DAG, e.g. a `Scan` both queries read when Stage 1 -/// shares it. The physical candidates are the product of the -/// options, here 2 × 2 = 4, less the assignments that put a build at -/// ingestion time over one that is not; Stage 3 prices each. -/// `at_ingestion` records why ingestion is ruled out when it is (e.g. -/// `Filtered`, `NotRepeated`). Each physical candidate carries the -/// resulting assignment ([`PhysicalWorkloadPlan::materialization`]). -#[derive(Debug, Clone)] -pub(crate) struct MaterializationCandidate { - /// The build in the implemented DAG. - pub(crate) node: Rc, - /// The queries (indices into the roots) that read it. - pub(crate) readers: Vec, - /// `Ok` when `MatAtIngestion` is an option. - pub(crate) at_ingestion: Result<(), IngestionPruneReason>, - /// Whether `MatAtQueryKeptAcrossEvaluations` is an option: the - /// deployment keeps query-time results and it is a tumbling-window chain - /// that builds one new tumbling window per evaluation. - pub(crate) may_keep_across_evaluations: bool, -} - -impl MaterializationCandidate { - /// Its options, `NotMaterialized` first: `MatAtIngestion` when eligible, - /// `MatAtQuery` when two or more queries read it, and - /// `MatAtQueryKeptAcrossEvaluations` when it may be kept across - /// evaluations. - pub(crate) fn options(&self) -> Vec { - let mut options = vec![Materialization::NotMaterialized]; - if self.at_ingestion.is_ok() { - options.push(Materialization::MatAtIngestion(Medium::Memory)); - } - if self.readers.len() > 1 { - options.push(Materialization::MatAtQuery(Medium::Memory)); - } - if self.may_keep_across_evaluations { - options.push(Materialization::MatAtQueryKeptAcrossEvaluations( - Medium::Memory, - )); - } - options - } -} - -/// Whether a tumbling-window chain of `tumbling_window_width_ms`-wide tumbling windows read by -/// queries repeating as `readers` builds exactly one new tumbling window per -/// evaluation: there is a reader, every reader repeats at a fixed interval -/// equal to the tumbling-window width, and the readers whose evaluation phase is known -/// share one phase (modulo the interval). Its other tumbling windows are the previous -/// evaluations' newest, so a deployment that keeps query-time results -/// reuses them (#509 Example 4's B3). -/// -/// Co-scheduling assumption: readers with the same interval and no known -/// phase run at the same instants as each other and as the readers with a -/// known phase, as plan selection (`asap-plan-selection`) assumes when it counts -/// one evaluation per interval. Readers at different known phases would -/// each need their own newest tumbling window, so they do not share one kept chain. -pub(crate) fn builds_one_tumbling_window_per_evaluation<'a>( - tumbling_window_width_ms: u64, - readers: impl IntoIterator, -) -> bool { - if tumbling_window_width_ms == 0 { - return false; - } - let mut phase_ms = None; - let mut any_reader = false; - for recurrence in readers { - any_reader = true; - if recurrence.fixed_interval_ms() != Some(tumbling_window_width_ms) { - return false; - } - if let QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - evaluation_phase, .. - }) = recurrence - { - let phase = evaluation_phase.0 % tumbling_window_width_ms; - if *phase_ms.get_or_insert(phase) != phase { - return false; - } - } - } - any_reader -} - -/// Stage 2 for one CandidateLogicalASAPDAG: implement every operator -/// (physical operator implementation), then one physical candidate per -/// materialization assignment of its mat_candidates, the all-`NotMaterialized` one -/// first. `entries[i]` is the workload entry of query `i` (`roots[i]`): -/// its recurrence and predictability decide whether state may be kept from -/// ingestion time; a missing entry is unknown. `id` is left empty for the -/// caller to name. Sharing between `roots` is preserved: one cache serves the -/// whole workload. `query_time_results_retained` says whether a query-time -/// result may be kept across evaluations. -/// -/// The reference pricer: only the developer tools (and tests) enumerate a -/// whole workload this way; planning goes through -/// [`generate_physical_asap_memo_candidates`]. -#[cfg(any(test, feature = "reference-pricer"))] -pub fn optimize_physical_asap( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result, PhysicalASAPOptimizationError> { - let (implemented, _, mat_candidates) = - implement_and_find_candidates(roots, entries, data, query_time_results_retained)?; - materialization_assignments(&mat_candidates) - .map(|assignment| { - assigned_candidate(from_logical, &implemented, &mat_candidates, assignment) - }) - .collect() -} - -/// Stage 2 for one CandidateLogicalASAPDAG and one materialization -/// assignment: the physical candidate [`optimize_physical_asap`] lists for -/// it. `choose(readers)` gives the option for each materialization -/// candidate (each summary build in discovery order, query by query, each -/// query's builds as a walk from its root meets them, then each highest -/// shared non-summary node in the same order), `readers` being the queries -/// that read it. `None` when -/// a chosen option is not one of the build's, or the assignment -/// materializes a build at ingestion time above one at query time. -#[cfg(feature = "test-support")] -pub fn optimize_physical_asap_assigned( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - choose: &mut dyn FnMut(&[usize]) -> Option, -) -> Result, PhysicalASAPOptimizationError> { - optimize_physical_asap_chosen( - from_logical, - roots, - entries, - data, - query_time_results_retained, - &mut |_, readers, _| choose(readers), - ) -} - -/// [`optimize_physical_asap_chosen`]'s choice per materialization -/// candidate: from the logical node it implements, its readers and its -/// options. `None` rejects the assignment. -pub(crate) type ChooseMaterialization<'a> = - dyn FnMut(*const OperatorNode, &[usize], &[Materialization]) -> Option + 'a; - -/// Stage 2 for one CandidateLogicalASAPDAG and one materialization -/// assignment chosen node by node: `choose(logical, readers, options)` -/// gives the option for each materialization candidate, `logical` being -/// the address of the logical node of `roots` it implements (null when it -/// implements none, e.g. a node the implementation adds) and `readers` the -/// queries that read it. `None` when `choose` rejects, a chosen option is -/// not one of the candidate's, or the assignment materializes a build at ingestion time -/// above one at query time. The search that decodes one plan from a model -/// solution uses it to build that plan alone, not every assignment. -pub(crate) fn optimize_physical_asap_chosen( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - choose: &mut ChooseMaterialization<'_>, -) -> Result, PhysicalASAPOptimizationError> { - let (implemented, cache, mat_candidates) = - implement_and_find_candidates(roots, entries, data, query_time_results_retained)?; - let logical_of: HashMap<*const OperatorNode, *const OperatorNode> = cache - .iter() - .map(|(logical, implemented)| (Rc::as_ptr(implemented), *logical)) - .collect(); - let mut assignment = Vec::with_capacity(mat_candidates.len()); - for mat_candidate in &mat_candidates { - let logical = logical_of - .get(&Rc::as_ptr(&mat_candidate.node)) - .copied() - .unwrap_or(std::ptr::null()); - let options = mat_candidate.options(); - let Some(m) = - choose(logical, &mat_candidate.readers, &options).filter(|m| options.contains(m)) - else { - return Ok(None); - }; - assignment.push(m); - } - let nodes: Vec<_> = mat_candidates.iter().map(|c| Rc::clone(&c.node)).collect(); - if !down_closed(&mat_candidates, &candidates_below(&nodes), &assignment) { - return Ok(None); - } - assigned_candidate(from_logical, &implemented, &mat_candidates, assignment).map(Some) -} - -/// The physical candidate of the implemented roots `implemented` with -/// `assignment[c]` chosen for `mat_candidates[c]`. -fn assigned_candidate( - from_logical: &str, - implemented: &[QueryRoot], - mat_candidates: &[MaterializationCandidate], - assignment: Vec, -) -> Result { - let materialized: Vec<_> = mat_candidates - .iter() - .zip(&assignment) - .filter(|(_, m)| **m != Materialization::NotMaterialized) - .map(|(mat_candidate, m)| (mat_candidate, *m)) - .collect(); - let mut options = MaterializationAssignment::with_default(Materialization::NotMaterialized); - for (mat_candidate, materialization) in &materialized { - options.set(&mat_candidate.node, *materialization); - } - let operators: Vec> = implemented - .iter() - .flat_map(|root| root.operator_refs().into_iter().cloned()) - .collect(); - let (split, options) = split_shared_by_timing(&operators, &options); - let replaced: HashMap<_, _> = operators - .iter() - .map(Rc::as_ptr) - .zip(split.iter().cloned()) - .collect(); - let split_roots = map_roots(implemented, &mut |node| { - Ok(replaced[&Rc::as_ptr(node)].clone()) - })?; - let mut timing = NodeToTimedNodeCache::new(); - let timed = map_roots(&split_roots, &mut |node| { - Ok(apply_materialization_timings(node, &options, &mut timing)?) - })?; - let dag = compile_physical_asap_workload_with_node_ids(&timed)?.dag; - let materialization = dag - .nodes - .iter() - .filter(|n| { - matches!( - n.payload, - Operator::ASAP( - ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } - ) - ) || n.materialization != Materialization::NotMaterialized - }) - .map(|n| (n.id, n.materialization)) - .collect(); - Ok(PhysicalWorkloadPlan { - id: String::new(), - from_logical: from_logical.to_string(), - roots: timed, - dag, - materialization, - }) -} - -/// `roots` implemented, the implementation of each logical node, and the -/// materialization candidates of the implemented roots. -fn implement_and_find_candidates( - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result< - ( - Vec, - NodeToImplementedNodeCache, - Vec, - ), - PhysicalASAPOptimizationError, -> { - let mut cache = HashMap::new(); - let implemented = map_roots(roots, &mut |node| implement(node, &mut cache))?; - let mat_candidates = - materialization_candidates(&implemented, entries, data, query_time_results_retained); - check_tumbling_windows_on_range_steps(&mat_candidates, entries)?; - Ok((implemented, cache, mat_candidates)) -} - -/// `Err` for the first tumbling-window chain whose tumbling-window width does not -/// divide the step of a reader's range evaluation -/// ([`PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep`]): that step -/// would end inside a tumbling window. -fn check_tumbling_windows_on_range_steps( - mat_candidates: &[MaterializationCandidate], - entries: &[QueryWorkloadEntry], -) -> Result<(), PhysicalASAPOptimizationError> { - for candidate in mat_candidates { - check_summary_range_steps(&candidate.node, &candidate.readers, entries)?; - } - Ok(()) -} - -fn check_summary_range_steps( - node: &OperatorNode, - readers: &[usize], - entries: &[QueryWorkloadEntry], -) -> Result<(), PhysicalASAPOptimizationError> { - let Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) = node.asap() - else { - return Ok(()); - }; - for &query in readers { - let Some(range) = entries - .get(query) - .and_then(|entry| entry.range_evaluation) - .filter(|range| range.evaluations_per_request() > 1) - else { - continue; - }; - if range.step_ms % tumbling_window_width_ms.max(&1) != 0 { - return Err( - PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep { - tumbling_window_width_ms: *tumbling_window_width_ms, - step_ms: range.step_ms, - }, - ); - } - } - Ok(()) -} - -/// Whether `node` is a summary build (`SummaryAgg` or `SummaryTumblingWindowChain`). -fn is_summary_build(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) -} - -/// The materialization candidates of `roots` (implemented), with their -/// readers, whether each may run at ingestion time and whether it may be -/// kept from query time across evaluations: every summary build, in -/// discovery order, then every highest shared non-summary node, in -/// discovery order. A shared node is one two or more queries read; the -/// highest are those a root or an unshared node reads (e.g. a `Scan` that -/// two windows read). Only a highest one is offered `MatAtQuery`, so a -/// shared sub-DAG adds one candidate, not one per node. -pub(crate) fn materialization_candidates( - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Vec { - let mut reached: Vec> = Vec::new(); - let mut readers: HashMap<*const OperatorNode, Vec> = HashMap::new(); - let mut consumers: HashMap<*const OperatorNode, Vec<*const OperatorNode>> = HashMap::new(); - let mut root_nodes: HashSet<*const OperatorNode> = HashSet::new(); - for (query, root) in roots.iter().enumerate() { - for operator in root.operator_refs() { - root_nodes.insert(Rc::as_ptr(operator)); - for node in OperatorNode::reachable(operator) { - let entry = readers.entry(Rc::as_ptr(&node)).or_default(); - if entry.is_empty() { - reached.push(Rc::clone(&node)); - for child in node.children() { - consumers - .entry(Rc::as_ptr(child)) - .or_default() - .push(Rc::as_ptr(&node)); - } - } - if !entry.contains(&query) { - entry.push(query); - } - } - } - } - let shared = |node: *const OperatorNode| readers[&node].len() > 1; - let highest_shared = |node: &Rc| { - let ptr = Rc::as_ptr(node); - shared(ptr) - && (root_nodes.contains(&ptr) - || consumers - .get(&ptr) - .is_some_and(|c| c.iter().any(|&consumer| !shared(consumer)))) - }; - let (nodes, shared_results): (Vec<_>, Vec<_>) = - reached.into_iter().partition(|n| is_summary_build(n)); - let below = candidates_below(&nodes); - let builds = nodes.iter().enumerate().map(|(c, node)| { - let readers = readers[&Rc::as_ptr(node)].clone(); - let eligibility = - summary_reader_eligibility(node, &readers, entries, data, query_time_results_retained); - let at_ingestion = eligibility.at_ingestion.and_then(|()| { - // Only this build and the builds below it at ingestion time. - let mut trial = - MaterializationAssignment::with_default(Materialization::NotMaterialized); - for &v in below[c].iter().chain([&c]) { - trial.set(&nodes[v], Materialization::MatAtIngestion(Medium::Memory)); - } - apply_materialization_timings(node, &trial, &mut NodeToTimedNodeCache::new()) - .map(|_| ()) - .map_err(|e| IngestionPruneReason::UpstreamAtQueryTime(e.to_string())) - }); - MaterializationCandidate { - node: Rc::clone(node), - readers, - at_ingestion, - may_keep_across_evaluations: eligibility.may_keep_across_evaluations, - } - }); - // A node fixed at ingestion time cannot be built at query time. - let shared_results = shared_results - .into_iter() - .filter(|n| highest_shared(n) && n.timing != Some(ExecutionTiming::IngestionTime)) - .map(|node| MaterializationCandidate { - readers: readers[&Rc::as_ptr(&node)].clone(), - node, - at_ingestion: Err(IngestionPruneReason::NotASummaryBuild), - may_keep_across_evaluations: false, - }); - builds.chain(shared_results).collect() -} - -/// Whether every scan below `node` reads a time series (a PromQL source). -fn reads_only_time_series(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), - _ => node.children().iter().all(|c| reads_only_time_series(c)), - } -} - -/// The phase, modulo `tumbling_window_width_ms` (which must be nonzero), of -/// every evaluation of `demand`, if known. A plain fixed interval has cadence but no phase, so -/// it is unknown. An explicit schedule has one only when every timestamp in -/// it has the same phase. -fn plan_tumbling_window_phase( - demand: &RepeatedDemand, - tumbling_window_width_ms: u64, -) -> Option { - match demand { - RepeatedDemand::FixedIntervalAt { - evaluation_phase, .. - } => Some(evaluation_phase.0 % tumbling_window_width_ms), - RepeatedDemand::Scheduled(times) => { - let first = times.first().map(|t| t.0 % tumbling_window_width_ms); - first.filter(|phase| { - times - .iter() - .all(|t| t.0 % tumbling_window_width_ms == *phase) - }) - } - RepeatedDemand::FixedInterval(_) | RepeatedDemand::EstimatedRate(_) => None, - } -} - -/// The workload rules of S4 for `node` (module doc). -fn ingestion_eligibility( - node: &OperatorNode, - readers: &[usize], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, -) -> Result<(), IngestionPruneReason> { - use IngestionPruneReason as Why; - if !matches!( - data.arrival, - DataArrival::Ingesting { .. } | DataArrival::Mixed { .. } - ) { - return Err(Why::NotIngesting); - } - let demands: Vec> = - readers.iter().map(|&q| entries.get(q)).collect(); - if demands - .iter() - .any(|d| !d.is_some_and(|d| matches!(d.recurrence, QueryRecurrence::Repeated(_)))) - { - return Err(Why::NotRepeated); - } - if demands - .iter() - .any(|d| !d.is_some_and(|d| matches!(d.predictability, Predictability::Predictable { .. }))) - { - return Err(Why::NotPredictable); - } - if is_filtered_build(node) { - return Err(Why::Filtered); - } - let (family, chain_tumbling_window_width_ms) = match node.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => (family, None), - Some(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - .. - }) => (family, Some(*tumbling_window_width_ms)), - _ => unreachable!("a materialization candidate is a summary build"), - }; - if matches!( - family, - FieldDataType::ExactAggregate(ExactKind::Rate | ExactKind::Increase | ExactKind::IRate, _) - ) { - // Assumption (module doc): continuously ingesting PromQL data - // arrives in time order per series, as Prometheus assumes. - let ordered = - matches!(data.arrival, DataArrival::Ingesting { .. }) && reads_only_time_series(node); - if !ordered { - return Err(Why::CounterNeedsTimeOrder); - } - } - if let Some(tumbling_window_width_ms) = chain_tumbling_window_width_ms { - // A chain's tumbling windows are built once each, on the grid anchored at the - // epoch (the plan carries no tumbling-window origin): every reader must - // evaluate on its boundaries. A reader at a fixed interval needs an - // interval the width divides, whatever its phase; a reader of known - // phase also needs that phase on the grid. A reader of unknown phase - // is taken to run at multiples of its interval (phase 0), as for a - // whole-window build below. - for d in demands.iter().flatten() { - let QueryRecurrence::Repeated(demand) = &d.recurrence else { - continue; - }; - if d.recurrence - .fixed_interval_ms() - .is_some_and(|interval_ms| !interval_ms.is_multiple_of(tumbling_window_width_ms)) - { - return Err(Why::TumblingWindowsOffInterval); - } - if tumbling_window_width_ms == 0 { - return Err(Why::TumblingWindowsOffPhase); - } - let Some(phase_ms) = plan_tumbling_window_phase(demand, tumbling_window_width_ms) - else { - if matches!(demand, RepeatedDemand::Scheduled(_)) { - // A schedule whose times are not all on one tumbling-window phase. - return Err(Why::TumblingWindowsOffPhase); - } - continue; - }; - if phase_ms != 0 { - return Err(Why::TumblingWindowsOffPhase); - } - } - return Ok(()); - } - if demands.iter().flatten().any(|d| { - d.range_evaluation - .is_some_and(|range| range.evaluations_per_request() > 1) - }) { - return Err(Why::RangeEvaluatedWholeWindow); - } - // A whole-window build: one tumbling window per evaluation, built (like - // a chain's tumbling windows) on the grid of its width anchored at the epoch, so - // every evaluation must end on that grid. The readers share one fixed - // interval and their declared phases agree; a reader with no declared - // phase is taken to evaluate at multiples of its interval (phase 0), as - // a chain's is, which lies on the grid once the interval does. - let mut interval_ms = None; - let mut phase_ms = None; - for d in demands.into_iter().flatten() { - let (interval, phase) = match &d.recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) => { - (u64::from(interval.0), None) - } - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval, - evaluation_phase, - }) => ( - u64::from(interval.0), - evaluation_phase.0.checked_rem(u64::from(interval.0)), - ), - _ => return Err(Why::WindowNotFixed), - }; - if interval == 0 { - return Err(Why::WindowNotFixed); - } - if *interval_ms.get_or_insert(interval) != interval { - return Err(Why::WindowNotFixed); - } - if let Some(phase) = phase { - if *phase_ms.get_or_insert(phase) != phase { - return Err(Why::WindowNotFixed); - } - } - } - let Some(interval_ms) = interval_ms else { - return Err(Why::WindowNotFixed); - }; - let Some(NonASAPOp::TimeRange { - range, - kind: asap_types::ir::TimeRangeKind::Range, - child, - }) = node.children()[0].non_asap() - else { - return Err(Why::WindowNotFixed); - }; - if matches!(child.non_asap(), Some(NonASAPOp::TimeShift { .. })) { - return Err(Why::WindowNotFixed); - } - let window_ms = u64::try_from(range.as_millis()) - .ok() - .filter(|&ms| ms > 0) - .ok_or(Why::WindowNotFixed)?; - let on_grid = |ms: u64| ms.is_multiple_of(window_ms); - if on_grid(interval_ms) && phase_ms.is_none_or(on_grid) { - Ok(()) - } else { - Err(Why::WindowOffEpochGrid) - } -} - -/// For each of `nodes`, the indices of the others below it. -fn candidates_below(nodes: &[Rc]) -> Vec> { - nodes - .iter() - .map(|node| { - let reachable = OperatorNode::reachable(node); - (0..nodes.len()) - .filter(|&v| { - !Rc::ptr_eq(&nodes[v], node) - && reachable.iter().any(|n| Rc::ptr_eq(n, &nodes[v])) - }) - .collect() - }) - .collect() -} - -/// Whether, below every ingestion-time build of `assignment`, every build -/// is at ingestion time too and every shared non-summary node is -/// `NotMaterialized`: ingestion work reads it as it arrives, and its -/// query-time readers read a copy ([`split_shared_by_timing`]). -/// Simplification: that copy is not offered `MatAtQuery` even when two -/// query-time readers remain. `below` is [`candidates_below`] of -/// `mat_candidates`. -fn down_closed( - mat_candidates: &[MaterializationCandidate], - below: &[Vec], - assignment: &[Materialization], -) -> bool { - let at_ingestion = |m: &Materialization| m.timing() == ExecutionTiming::IngestionTime; - assignment.iter().enumerate().all(|(c, m)| { - !at_ingestion(m) - || below[c].iter().all(|&v| { - if is_summary_build(&mat_candidates[v].node) { - at_ingestion(&assignment[v]) - } else { - assignment[v] == Materialization::NotMaterialized - } - }) - }) -} - -/// Every assignment of an option to each materialization candidate (mixed -/// radix, the last varying fastest, all `NotMaterialized` first) that is -/// [`down_closed`]. -#[cfg(any(test, feature = "reference-pricer"))] -fn materialization_assignments( - mat_candidates: &[MaterializationCandidate], -) -> impl Iterator> + '_ { - let options: Vec> = mat_candidates.iter().map(|c| c.options()).collect(); - let nodes: Vec<_> = mat_candidates.iter().map(|c| Rc::clone(&c.node)).collect(); - let below = candidates_below(&nodes); - let mut digits = Some(vec![0usize; mat_candidates.len()]); - std::iter::from_fn(move || loop { - let current = digits.as_mut()?; - let assignment: Vec = current - .iter() - .zip(&options) - .map(|(&d, options)| options[d]) - .collect(); - // Advance (mixed radix); `None` after the last assignment. - let mut carried = true; - for (digit, options) in current.iter_mut().zip(&options).rev() { - *digit += 1; - if *digit < options.len() { - carried = false; - break; - } - *digit = 0; - } - if carried { - digits = None; - } - if down_closed(mat_candidates, &below, &assignment) { - return Some(assignment); - } - digits.as_ref()?; - }) -} - -/// `roots` with every operator node they read replaced by `f`'s result. -fn map_roots( - roots: &[QueryRoot], - f: &mut impl FnMut(&Rc) -> Result, PhysicalASAPOptimizationError>, -) -> Result, PhysicalASAPOptimizationError> { - roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => Ok(QueryRoot::Operator(f(node)?)), - QueryRoot::Scalar(expr) => { - let mut mapped = HashMap::new(); - for node in expr.operator_refs() { - mapped.insert(Rc::as_ptr(node), f(node)?); - } - Ok(QueryRoot::Scalar(expr.map_operator_refs(&mut |node| { - mapped[&Rc::as_ptr(node)].clone() - }))) - } - }) - .collect() -} - -/// Lower the private operators of one candidate while preserving its already -/// canonical input-group nodes. This reuses the complete-plan implementation -/// rules (currently exact top-k -> Project/Limit/Sort), without assigning timing -/// or materialization and without recursively implementing another group's input. -/// Returned new operators must be enumerated and priced before plan selection. -pub(crate) fn implement_candidate_operators( - root: &Rc, - inputs: &[Rc], -) -> Result, PhysicalASAPOptimizationError> { - // The inventory rejects a root aliasing an input; an input outside the - // subtree is unused here and rejected by the inventory as `InvalidMemo`. - debug_assert!(!inputs.iter().any(|input| Rc::ptr_eq(input, root))); - let mut implementations = inputs - .iter() - .map(|input| (Rc::as_ptr(input), Rc::clone(input))) - .collect(); - implement(root, &mut implementations) -} - -/// Each node's implementation. -type NodeToImplementedNodeCache = HashMap<*const OperatorNode, Rc>; - -fn implement( - node: &Rc, - cache: &mut NodeToImplementedNodeCache, -) -> Result, PhysicalASAPOptimizationError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - implement(child, cache)?; - } - let implemented = match node.non_asap() { - Some(NonASAPOp::Aggregate { measures, .. }) - if measures.iter().any(|m| matches!(m, AggIntent::TopK { .. })) => - { - implement_topk(node, cache)? - } - _ => rebuilt(node, cache)?, - }; - cache.insert(Rc::as_ptr(node), implemented.clone()); - Ok(implemented) -} - -/// Exact top-k as `Limit(Sort)` per group, which outputs the ranked rows of -/// the child, under a `Project` restoring the logical `[keys.., topk_k]` -/// output (the ranked value under `topk_k`): a root returns those columns, -/// and a node reading the top-k resolved its columns against them. -fn implement_topk( - node: &Rc, - cache: &NodeToImplementedNodeCache, -) -> Result, PhysicalASAPOptimizationError> { - let unsupported = |why: &str| PhysicalASAPOptimizationError::ExactTopK(why.to_string()); - let Some(NonASAPOp::Aggregate { - reduction, - measures, - filters, - having, - child, - .. - }) = node.non_asap() - else { - unreachable!("implement_topk is called on an Aggregate") - }; - let Reduction::Reduce(keys) = reduction else { - return Err(unsupported("per-entity reduction")); - }; - let [AggIntent::TopK { k, .. }] = measures.as_slice() else { - return Err(unsupported("measures besides the top-k")); - }; - if filters.iter().any(Option::is_some) { - return Err(unsupported("filtered measure")); - } - if having.is_some() { - return Err(unsupported("HAVING")); - } - let child = cache[&Rc::as_ptr(child)].clone(); - let value = asap_types::ir::schema::topk::ranked_column(&child) - .map_err(|e| unsupported(&format!("no ranked column: {e}")))?; - let sorted = Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(value), - ascending: false, - nulls_first: false, - }], - partition_by: keys.clone(), - child: child.clone(), - }))? - .with_guarantee(child.guarantee.clone()), - ); - let limited = Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Limit { - n: Some(*k), - offset: 0, - partition_by: keys.clone(), - child: sorted, - }))? - .with_guarantee(node.guarantee.clone()), - ); - let output_name = node.schema.fields[keys.len()].name.clone(); - let cols = asap_types::ir::schema::topk::ranked_row_projection( - &child, - &Reduction::Reduce(keys.clone()), - &output_name, - ) - .map_err(|error| unsupported(&format!("ranked rows: {error}")))?; - Ok(Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: limited, - }))? - .with_guarantee(node.guarantee.clone()), - )) -} - -/// `node` over its implemented children; the same `Rc` when none changed. -fn rebuilt( - node: &Rc, - cache: &NodeToImplementedNodeCache, -) -> Result, PhysicalASAPOptimizationError> { - let input = |child: &Rc| cache[&Rc::as_ptr(child)].clone(); - let changed = node - .children() - .iter() - .any(|child| !Rc::ptr_eq(child, &input(child))); - Ok(if changed { - Rc::new( - node.with_new_children(|child| input(child))? - .with_guarantee(node.guarantee.clone()), - ) - } else { - node.clone() - }) -} - -/// Stage 2 internals that other crates' tests call: building one physical -/// candidate for a chosen assignment, the catalog steps, and the timing split. -#[cfg(feature = "test-support")] -pub mod test_support { - use asap_types::ir::QueryRoot; - use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; +//! that output and builds the plan. The crate depends on no other stage +//! crate. - use crate::{PhysicalASAPOptimizationError, PhysicalWorkloadPlan}; - - pub use crate::catalog::test_support as catalog; - pub use crate::split_by_timing::split_shared_by_timing; - - /// See `optimize_physical_asap_chosen` in this crate. - pub fn optimize_physical_asap_chosen( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - choose: &mut crate::ChooseMaterialization<'_>, - ) -> Result, PhysicalASAPOptimizationError> { - crate::optimize_physical_asap_chosen( - from_logical, - roots, - entries, - data, - query_time_results_retained, - choose, - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_logical_asap_optimization::test_support::lower_promql; - use asap_types::ir::physical_export::PhysicalASAPNodeId; - use asap_types::ir::physical_export::PhysicalASAPOperatorPayload as Payload; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::properties::ExecutionTiming; - use asap_types::ir::ASAPOp; - - // Candidate-local implementation must leave another group's canonical root untouched. - #[test] - fn candidate_operator_lowering_preserves_input_boundaries() { - let root = lower_promql( - "topk by (job) (3, sum_over_time(m[1m]))", - AccuracyTarget::exact(), - ); - let boundary = Rc::clone(root.children()[0]); - let lowered = - implement_candidate_operators(&root, std::slice::from_ref(&boundary)).unwrap(); - assert_eq!(lowered.schema, root.schema); - let nodes = OperatorNode::reachable(&lowered); - assert!(nodes.iter().any(|node| node.operator.kind_name() == "Sort")); - assert!(nodes - .iter() - .any(|node| node.operator.kind_name() == "Limit")); - assert!(nodes.iter().any(|node| Rc::ptr_eq(node, &boundary))); - } - - // Logical and exact ranked rows retain the full series identity contract. - #[test] - fn topk_logical_contract_preserves_series_identity() { - let template = lower_promql("topk(3, sum_over_time(m[1m]))", AccuracyTarget::exact()); - let source = asap_logical_asap_optimization::test_support::metric_scan(&[ - "job", - "instance", - asap_types::ir::schema::PROMQL_SERIES_IDENTITY, - ]); - let root = Rc::new(template.with_new_children(|_| Rc::clone(&source)).unwrap()); - let boundary = Rc::clone(root.children()[0]); - let lowered = implement_candidate_operators(&root, &[boundary]).unwrap(); - assert_eq!(lowered.schema, root.schema); - assert_eq!( - names(&root), - [ - "topk_3", - "job", - "instance", - asap_types::ir::schema::PROMQL_SERIES_IDENTITY - ] - ); - assert!(root - .schema - .fields - .iter() - .any(|field| field.name == asap_types::ir::schema::PROMQL_SERIES_IDENTITY)); - assert_eq!( - root.schema.fields[0].dtype, - asap_types::ir::schema::FieldDataType::Plain(asap_types::ir::schema::DataType::Float64) - ); - } - - /// The all-`NotMaterialized` candidate, with no workload declared. - fn first_candidate(roots: &[QueryRoot]) -> PhysicalWorkloadPlan { - optimize_physical_asap("L1", roots, &[], &DataWorkload::default(), false) - .unwrap() - .remove(0) - } - - fn operator_root(candidate: &PhysicalWorkloadPlan, i: usize) -> Rc { - match &candidate.roots[i] { - QueryRoot::Operator(node) => node.clone(), - QueryRoot::Scalar(_) => panic!("operator root"), - } - } - - fn names(node: &OperatorNode) -> Vec { - node.schema.fields.iter().map(|f| f.name.clone()).collect() - } - - /// The `Limit` under an exact top-k's restoring `Project`. - fn topk_limit(project: &OperatorNode) -> Rc { - let NonASAPOp::Project { child, .. } = project.expect_non_asap() else { - panic!("project, found {}", project.operator.kind_name()) - }; - child.clone() - } - - /// The single sort key of the `Sort` under the `Limit` at `limit`. - fn topk_sort_key(limit: &OperatorNode) -> (ScalarExpr, bool) { - let NonASAPOp::Limit { child: sort, .. } = limit.expect_non_asap() else { - panic!("limit, found {}", limit.operator.kind_name()) - }; - let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { - panic!("sort below limit") - }; - let [key] = keys.as_slice() else { - panic!("one sort key") - }; - (key.expr.clone(), key.ascending) - } - - /// Example 1's shape, `topk by (job) (k, per-series rows)`, sorts each - /// group by the sample value, descending. - #[test] - fn promql_topk_over_series_sorts_by_value_descending() { - let root = lower_promql( - "topk by (job) (10, sum_over_time(m[1m]))", - AccuracyTarget::exact(), - ); - let candidate = first_candidate(&[root.clone().into()]); - let physical = operator_root(&candidate, 0); - assert_eq!(names(&physical), names(&root)); - let limit = topk_limit(&physical); - assert_eq!(limit.schema.fields[1].name, "value"); - assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); - } - - /// `topk(k, sum by (job) (..))` sorts by the summed measure. - #[test] - fn promql_topk_over_grouped_aggregate_sorts_by_its_measure() { - let root = lower_promql("topk(5, sum by (job) (m))", AccuracyTarget::exact()); - let candidate = first_candidate(&[root.into()]); - let physical = operator_root(&candidate, 0); - // The answers keep their `job` label. - assert_eq!(names(&physical), ["topk_5", "job"]); - let limit = topk_limit(&physical); - assert_eq!(names(&limit), ["job", "sum"]); - assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); - } - - /// A node reading an exact top-k keeps resolving the columns it bound - /// against the logical `[keys.., topk_k]` output: `sum by (job)` still - /// groups by `job`, not by whatever column now sits at its index. - #[test] - fn consumer_of_exact_topk_reads_its_logical_columns() { - let root = lower_promql( - "sum by (job) (topk by (job) (10, sum_over_time(m[1m])))", - AccuracyTarget::exact(), - ); - let candidate = first_candidate(&[root.clone().into()]); - let physical = operator_root(&candidate, 0); - assert_eq!(names(&physical), names(&root)); - let NonASAPOp::Aggregate { child: input, .. } = physical.expect_non_asap() else { - panic!("sum root") - }; - let NonASAPOp::Project { - cols, child: limit, .. - } = input.expect_non_asap() - else { - panic!("project over the top-k rows") - }; - let read: Vec<_> = cols - .iter() - .map(|item| { - let ScalarExpr::Column(c) = item.expr else { - panic!("column") - }; - ( - item.alias.clone().unwrap(), - limit.schema.fields[c].name.clone(), - ) - }) - .collect(); - assert_eq!( - read, - [ - ("job".to_string(), "job".to_string()), - ("topk_10".to_string(), "value".to_string()) - ] - ); - assert_eq!(topk_sort_key(limit), (ScalarExpr::Column(1), false)); - } - - /// An `Aggregate{[TopK]}` that Sort + Limit cannot express (another - /// measure, `HAVING`) fails, so its candidate is pruned with the reason. - #[test] - fn topk_with_extra_measure_or_having_fails() { - use asap_logical_asap_optimization::test_support::{aggregate, metric_scan}; - use asap_types::ir::scalar::expr_ir::ScalarValue; - use asap_types::ir::Predicate; - let topk = AggIntent::TopK { - k: 3, - accuracy: AccuracyTarget::exact(), - }; - let count = AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }; - let by_job = || Reduction::by(vec![2]); - let cases = [ - aggregate( - by_job(), - vec![topk.clone(), count], - vec![], - None, - metric_scan(&["job"]), - ), - aggregate( - by_job(), - vec![topk], - vec![], - Some(Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true)))), - metric_scan(&["job"]), - ), - ]; - for root in cases { - let error = - optimize_physical_asap("L1", &[root.into()], &[], &DataWorkload::default(), false) - .unwrap_err(); - assert!( - matches!(error, PhysicalASAPOptimizationError::ExactTopK(_)), - "{error}" - ); - } - } - - /// Exact `topk by (job)` becomes Limit(Sort) partitioned by `job`, and a - /// child shared by two roots stays one node (one `Rc`, one DAG node). - #[test] - fn exact_topk_becomes_sort_then_limit_and_keeps_shared_child() { - let exact = AccuracyTarget::exact(); - let topk = lower_promql("topk by (job) (10, sum_over_time(m[1m]))", exact.clone()); - let sum = lower_promql("sum by (job) (sum_over_time(m[1m]))", exact); - // Share Q2's per-series child with Q1, as Stage 1 sharing would. - let NonASAPOp::Aggregate { child: shared, .. } = topk.expect_non_asap() else { - panic!("topk aggregate") - }; - let sum = Rc::new(sum.with_new_children(|_| shared.clone()).unwrap()); - let candidate = first_candidate(&[topk.into(), sum.into()]); - let root = |i: usize| match &candidate.roots[i] { - QueryRoot::Operator(node) => node.clone(), - QueryRoot::Scalar(_) => panic!("operator root"), - }; - - let NonASAPOp::Limit { - n: Some(10), - partition_by, - child: sort, - .. - } = topk_limit(&root(0)).expect_non_asap().clone() - else { - panic!("limit under the root") - }; - assert!(!partition_by.keys().is_empty()); - let NonASAPOp::Sort { - child: below_sort, - partition_by: sort_partition, - .. - } = sort.expect_non_asap() - else { - panic!("sort below limit") - }; - assert_eq!(*sort_partition, partition_by); - let NonASAPOp::Aggregate { - child: below_sum, .. - } = root(1).expect_non_asap().clone() - else { - panic!("sum root") - }; - assert!(Rc::ptr_eq(below_sort, &below_sum)); - - let dag = &candidate.dag; - let kind = |id: PhysicalASAPNodeId| match &dag - .nodes - .iter() - .find(|n| n.id == id) - .unwrap() - .payload - { - Payload::NonASAP(NonASAPOp::Sort { .. }) => "sort", - Payload::NonASAP(NonASAPOp::Limit { .. }) => "limit", - _ => "other", - }; - let producer_of = |consumer| { - dag.edges - .iter() - .find(|e| e.consumer == consumer) - .unwrap() - .producer - }; - let limit_id = producer_of(dag.roots[0]); - assert_eq!(kind(limit_id), "limit"); - let sort_id = producer_of(limit_id); - assert_eq!(kind(sort_id), "sort"); - assert_eq!(producer_of(sort_id), producer_of(dag.roots[1])); - } - - /// With summaries chosen, every node, the summary build included, runs at - /// query time. - #[test] - fn everything_runs_at_query_time() { - let root = lower_promql( - "topk by (job) (10, sum_over_time(m[1m]))", - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - // A summary (the last candidate) for every target. - let choice: Vec<_> = inventory - .targets - .iter() - .map(|t| t.choice_count() - 1) - .collect(); - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap() - .into_iter() - .collect(); - let candidate = first_candidate(&roots); - assert!(candidate - .dag - .nodes - .iter() - .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryAgg { .. })))); - assert!(candidate - .dag - .nodes - .iter() - .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); - } - - // ── Materialization ────────────────────────────────────────────────── - - use asap_types::workload::{DataArrival, RepetitionInterval, TimestampMs}; - - const MINUTE: u64 = 60_000; - - /// The composed logical candidate of `query` that builds its last - /// summary option (DDSketch for a quantile) for every target. - fn summary_root(query: &str) -> QueryRoot { - let root = lower_promql( - query, - AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(asap_types::ir::properties::AccuracyMetric::Rank), - ); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let choice: Vec<_> = inventory - .targets - .iter() - .map(|t| t.choice_count() - 1) - .collect(); - asap_logical_asap_optimization::test_support::compose_logical_candidate(&inventory, &choice) - .unwrap() - .remove(0) - } - - /// `root` with its summary build in 1-min tumbling windows. - fn in_tumbling_windows(root: QueryRoot) -> QueryRoot { - let roots = [root]; - let rule_candidates = - asap_logical_asap_optimization::test_support::tumbling_window_candidates( - &roots, - &[Some(MINUTE)], - ); - assert_eq!(rule_candidates.len(), 1, "{rule_candidates:?}"); - let mut pass2_candidates = - asap_logical_asap_optimization::test_support::window_composition_pass2_candidates( - &roots, - &rule_candidates, - ) - .unwrap(); - pass2_candidates.pop().unwrap().roots.unwrap().remove(0) - } - - const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - - /// The options `candidate` chose other than `NotMaterialized`, by node. - fn materialized(candidate: &PhysicalWorkloadPlan) -> Vec { - candidate - .materialization - .values() - .copied() - .filter(|m| *m != Materialization::NotMaterialized) - .collect() - } - - // An unselected unpredictable query must not restrict an ingestion candidate. - #[test] - fn selected_reader_eligibility_ignores_optional_unpredictable_consumers() { - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let node = root - .operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .unwrap(); - let entries = vec![ - every_minute(), - QueryWorkloadEntry { - predictability: Predictability::AdHoc, - ..every_minute() - }, - ]; - let data = arriving(); - let query_time_results_retained = false; - let selected = summary_materialization_eligibility( - &node, - &[0, 0], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert_eq!(selected.at_ingestion, Ok(())); - assert!(matches!( - summary_materialization_eligibility(&node, &[0, entries.len()], &entries, &data, query_time_results_retained), - Err(SummaryEligibilityError::UnknownReader { query }) if query == entries.len() - )); - let combined = summary_materialization_eligibility( - &node, - &[0, 1], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert_eq!( - combined.at_ingestion, - Err(IngestionPruneReason::NotPredictable) - ); - // The complete-DAG path uses the same selected-reader workload rules. - let candidates = - materialization_candidates(&[root], &entries, &data, query_time_results_retained); - let original = candidates - .iter() - .find(|candidate| { - matches!( - candidate.node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .unwrap(); - assert_eq!(original.at_ingestion, selected.at_ingestion); - assert_eq!( - original.may_keep_across_evaluations, - selected.may_keep_across_evaluations - ); - } - - // Phase and range-step restrictions apply only to the selected reader set. - #[test] - fn selected_reader_eligibility_checks_phase_and_range_compatibility() { - use asap_types::workload::RangeEvaluation; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let node = root - .operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .unwrap(); - let at = |phase| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(phase), - }), - ..every_minute() - }; - let entries = vec![ - at(0), - at(MINUTE / 2), - QueryWorkloadEntry { - range_evaluation: Some(RangeEvaluation { - step_ms: MINUTE / 2, - span_ms: 10 * MINUTE, - }), - ..at(0) - }, - ]; - let query_time_results_retained = true; - let data = arriving(); - let one = summary_materialization_eligibility( - &node, - &[0], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert!(one.may_keep_across_evaluations); - assert!(one.range_compatibility.is_ok()); - let different_phase = summary_materialization_eligibility( - &node, - &[0, 1], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert!(!different_phase.may_keep_across_evaluations); - assert_eq!( - different_phase.at_ingestion, - Err(IngestionPruneReason::TumblingWindowsOffPhase) - ); - let off_grid = summary_materialization_eligibility( - &node, - &[0, 2], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert!(matches!( - off_grid.range_compatibility, - Err(PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep { .. }) - )); - assert!(matches!( - summary_materialization_eligibility( - &node, - &[], - &entries, - &data, - query_time_results_retained - ), - Err(SummaryEligibilityError::NoSelectedReaders) - )); - } - - /// A predictable query evaluated every minute. - fn every_minute() -> QueryWorkloadEntry { - QueryWorkloadEntry { - predictability: Predictability::Predictable { known_at: None }, - ..asap_logical_asap_optimization::test_support::entry(QueryRecurrence::Repeated( - RepeatedDemand::FixedInterval(RepetitionInterval(MINUTE as u32)), - )) - } - } - - fn arriving() -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000.0, - }, - ..Default::default() - } - } - - /// The distinct nodes of `kind` the candidate's roots read. - fn node_of(candidate: &PhysicalWorkloadPlan, kind: &str) -> Vec> { - let mut nodes: Vec> = Vec::new(); - for root in candidate.roots.iter().flat_map(QueryRoot::operator_refs) { - for node in OperatorNode::reachable(root) { - if node.operator.kind_name() == kind && !nodes.iter().any(|n| Rc::ptr_eq(n, &node)) - { - nodes.push(node); - } - } - } - nodes - } - - /// #509 Example 4, B1 and B2: a tumbling-window chain read by a repeating, - /// predictable query over arriving data is either rebuilt at query time - /// or maintained at ingestion time. Maintained, the chain and the scan - /// below it run at ingestion time, and the merge and estimate at query - /// time. - #[test] - fn a_tumbling_window_chain_over_arriving_data_may_be_maintained_at_ingestion_time() { - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let candidates = - optimize_physical_asap("L1", &[root], &[every_minute()], &arriving(), false).unwrap(); - assert_eq!(candidates.len(), 2); - let [b2, b1] = &candidates[..] else { - unreachable!() - }; - assert!(!b2.materializes_any()); - assert!(b2 - .dag - .nodes - .iter() - .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); - assert_eq!(materialized(b1), [AT_INGESTION]); - let [chain] = &node_of(b1, "SummaryTumblingWindowChain")[..] else { - panic!("one chain") - }; - assert_eq!( - chain.materialization, - Materialization::MatAtIngestion(Medium::Memory) - ); - for below in OperatorNode::reachable(chain) { - assert_eq!(below.timing, Some(ExecutionTiming::IngestionTime)); - } - for kind in ["SummaryMerge", "SummaryEstimate"] { - for node in node_of(b1, kind) { - assert_eq!(node.timing, Some(ExecutionTiming::QueryTime), "{kind}"); - } - } - } - - /// A chain of 1-min tumbling windows, anchored at the epoch, read by two queries - /// every minute is maintained at ingestion time when they evaluate on - /// its tumbling-window boundaries (phases 0 and 2 min, or one phase unknown), not - /// when one is at phase 30 s: each tumbling window is built once, so it would read - /// tumbling windows 30 s off. - #[test] - fn a_tumbling_window_chain_is_maintained_only_on_its_readers_phases() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let at = |phase_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(phase_ms), - }), - ..every_minute() - }; - let at_ingestion = |demands: [QueryWorkloadEntry; 2]| { - let mut cache = HashMap::new(); - // One chain read by both queries. - let implemented = map_roots(&[root.clone(), root.clone()], &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &demands, &arriving(), false) - .into_iter() - .find(|c| { - matches!( - c.node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .expect("the chain is a materialization candidate") - .at_ingestion - }; - assert_eq!(at_ingestion([at(0), at(2 * MINUTE)]), Ok(())); - assert_eq!(at_ingestion([at(0), every_minute()]), Ok(())); - assert_eq!( - at_ingestion([at(0), at(30_000)]), - Err(Why::TumblingWindowsOffPhase) - ); - assert_eq!( - at_ingestion([at(30_000), every_minute()]), - Err(Why::TumblingWindowsOffPhase) - ); - } - - /// A chain of 1-min tumbling windows, anchored at the epoch, read every - /// 90 s ends a window at only every other evaluation (#1173), with or - /// without a declared phase on the grid: it is maintained at ingestion - /// time only when every reader's interval is a multiple of its width. - #[test] - fn a_tumbling_window_chain_is_maintained_only_on_its_readers_intervals() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let every = |interval_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - RepetitionInterval(interval_ms as u32), - )), - ..every_minute() - }; - let every_at_phase_0 = |interval_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(interval_ms as u32), - evaluation_phase: TimestampMs(0), - }), - ..every_minute() - }; - let at_ingestion = |demands: [QueryWorkloadEntry; 2]| { - let mut cache = HashMap::new(); - // One chain read by both queries. - let implemented = map_roots(&[root.clone(), root.clone()], &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &demands, &arriving(), false) - .into_iter() - .find(|c| { - matches!( - c.node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .expect("the chain is a materialization candidate") - .at_ingestion - }; - assert_eq!(at_ingestion([every_minute(), every(2 * MINUTE)]), Ok(())); - assert_eq!( - at_ingestion([every_minute(), every(90_000)]), - Err(Why::TumblingWindowsOffInterval) - ); - assert_eq!( - at_ingestion([every_minute(), every_at_phase_0(90_000)]), - Err(Why::TumblingWindowsOffInterval) - ); - } - - /// S4: maintaining state at ingestion time needs arriving data and - /// queries that repeat and are predictable; otherwise the chain only - /// runs at query time, and the materialization candidate says why. - #[test] - fn ingestion_time_needs_arriving_data_and_repeated_predictable_queries() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let at_rest = DataWorkload { - arrival: DataArrival::AtRest, - ..Default::default() - }; - let unknown = QueryWorkloadEntry { - predictability: Predictability::Unknown, - ..every_minute() - }; - let once = QueryWorkloadEntry { - recurrence: QueryRecurrence::OneTime { - invocations: 1, - execute_at: None, - }, - ..every_minute() - }; - for (demand, data, why) in [ - (every_minute(), at_rest, Why::NotIngesting), - (every_minute(), DataWorkload::default(), Why::NotIngesting), - (unknown, arriving(), Why::NotPredictable), - (once, arriving(), Why::NotRepeated), - ] { - let roots = [root.clone()]; - let candidates = - optimize_physical_asap("L1", &roots, std::slice::from_ref(&demand), &data, false) - .unwrap(); - assert_eq!(candidates.len(), 1, "{why:?}"); - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - let mat_candidates = materialization_candidates(&implemented, &[demand], &data, false); - assert_eq!(mat_candidates.len(), 1); - assert_eq!(mat_candidates[0].at_ingestion, Err(why)); - } - } - - /// A counter state (`rate`, `increase`) needs each series' samples in - /// time order. Continuously ingesting time-series (PromQL) data is taken - /// to arrive in that order, as Prometheus assumes, so its counter tumbling windows - /// may be maintained at ingestion time. - #[test] - fn counter_tumbling_windows_over_ingesting_promql_data_run_at_ingestion_time() { - for query in ["rate(m[5m])", "increase(m[5m])"] { - let roots = [in_tumbling_windows(summary_root(query))]; - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - let mat_candidates = - materialization_candidates(&implemented, &[every_minute()], &arriving(), false); - assert_eq!(mat_candidates[0].at_ingestion, Ok(()), "{query}"); - let candidates = - optimize_physical_asap("L1", &roots, &[every_minute()], &arriving(), false) - .unwrap(); - assert_eq!(candidates.len(), 2, "{query}"); - } - } - - /// Only continuously ingesting time-series data is taken to arrive in - /// time order: a counter over a table source, or over mixed data (whose - /// at-rest part may be loaded in any order), stays at query time. - #[test] - fn counter_tumbling_windows_over_other_sources_stay_at_query_time() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("rate(m[5m])")); - let QueryRoot::Operator(node) = &root else { - panic!("{root:?}") - }; - let table = QueryRoot::Operator(to_table(node)); - let mixed = DataWorkload { - arrival: DataArrival::Mixed { - rows_per_second: 1_000.0, - }, - ..Default::default() - }; - for (root, data) in [(table, arriving()), (root.clone(), mixed)] { - let roots = [root]; - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - let mat_candidates = - materialization_candidates(&implemented, &[every_minute()], &data, false); - assert_eq!( - mat_candidates[0].at_ingestion, - Err(Why::CounterNeedsTimeOrder) - ); - } - } - - /// `node` with every time-series scan turned into a table scan of the - /// same name and schema. - fn to_table(node: &Rc) -> Rc { - let mut out = (**node).clone(); - out.operator = node.operator.map_children(to_table); - if let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &mut out.operator { - if let Source::TimeSeries { metric } = source { - *source = Source::Table { - table_ref: metric.clone(), - }; - } - } - Rc::new(out) - } - - /// A whole-window build is maintained only for one window per - /// evaluation on the epoch grid of its width, where the backend builds - /// it: a fixed interval that is a multiple of the window, a phase (if - /// known) on that grid, and no filter. `m[1m]` every minute with no - /// declared phase is one tumbling 1-minute window per evaluation, taken - /// to run on the grid like a chain's reader of unknown phase; at phase - /// 30 s its windows would end off the grid, and every 30 s or every - /// 90 s some evaluations would. - #[test] - fn a_whole_window_build_needs_evaluations_on_the_epoch_grid_of_its_window() { - use IngestionPruneReason as Why; - let root = summary_root("quantile_over_time(0.99, m[1m])"); - let at_phase = |interval_ms: u32, phase_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(interval_ms), - evaluation_phase: TimestampMs(phase_ms), - }), - ..every_minute() - }; - let at = |interval_ms: u32| at_phase(interval_ms, 0); - let mat_candidates = |root: &QueryRoot, demand: QueryWorkloadEntry| { - let mut cache = HashMap::new(); - let implemented = map_roots(std::slice::from_ref(root), &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &[demand], &arriving(), false) - .remove(0) - .at_ingestion - }; - assert_eq!(mat_candidates(&root, at(MINUTE as u32)), Ok(())); - assert_eq!(mat_candidates(&root, every_minute()), Ok(())); - assert_eq!(mat_candidates(&root, at(2 * MINUTE as u32)), Ok(())); - assert_eq!( - mat_candidates(&root, at_phase(MINUTE as u32, 3 * MINUTE)), - Ok(()) - ); - assert_eq!( - mat_candidates(&root, at_phase(MINUTE as u32, 30_000)), - Err(Why::WindowOffEpochGrid) - ); - assert_eq!( - mat_candidates(&root, at(30_000)), - Err(Why::WindowOffEpochGrid) - ); - assert_eq!( - mat_candidates(&root, at(90_000)), - Err(Why::WindowOffEpochGrid) - ); - // Several readers of one build: a declared phase on the grid and an - // undeclared one agree; two declared phases, or two intervals, do not. - let mat_candidates_of = |demands: &[QueryWorkloadEntry]| { - let roots = vec![root.clone(); demands.len()]; - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - materialization_candidates(&implemented, demands, &arriving(), false) - .into_iter() - .find(|c| matches!(c.node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("the whole-window build is a materialization candidate") - .at_ingestion - }; - assert_eq!( - mat_candidates_of(&[at_phase(MINUTE as u32, MINUTE), every_minute()]), - Ok(()) - ); - assert_eq!( - mat_candidates_of(&[ - at(MINUTE as u32), - at_phase(MINUTE as u32, 2 * MINUTE + MINUTE / 2) - ]), - Err(Why::WindowNotFixed) - ); - assert_eq!( - mat_candidates_of(&[at(MINUTE as u32), at(2 * MINUTE as u32)]), - Err(Why::WindowNotFixed) - ); - let candidates = optimize_physical_asap( - "L1", - std::slice::from_ref(&root), - &[at(MINUTE as u32)], - &arriving(), - false, - ) - .unwrap(); - assert_eq!(materialized(&candidates[1]), [AT_INGESTION]); - - // The same build with a filter. - let QueryRoot::Operator(node) = &root else { - unreachable!() - }; - let build = OperatorNode::reachable(node) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .unwrap(); - let Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - grouping, - .. - }) = build.operator.clone() - else { - unreachable!() - }; - let filtered = Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - grouping, - filter: Some(asap_types::ir::Predicate(ScalarExpr::Literal( - asap_types::ir::scalar::expr_ir::ScalarValue::Boolean(true), - ))), - }), - build.schema.clone(), - ) - .with_guarantee(build.guarantee.clone()), - ); - assert_eq!( - mat_candidates(&QueryRoot::Operator(filtered), at(MINUTE as u32)), - Err(Why::Filtered) - ); - } - - /// A whole-window build read by a range evaluation of several steps per - /// request is not maintained at ingestion time: it would hold one - /// window per request, not one per step. A one-step range (`span_ms` - /// 0) is one evaluation per request and stays eligible. - #[test] - fn a_range_evaluated_whole_window_build_stays_at_query_time() { - use asap_types::workload::RangeEvaluation; - let root = summary_root("quantile_over_time(0.99, m[1m])"); - let at_minute = |span_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(0), - }), - range_evaluation: Some(RangeEvaluation { - step_ms: MINUTE, - span_ms, - }), - ..every_minute() - }; - let at_ingestion = |entry: QueryWorkloadEntry| { - let mut cache = HashMap::new(); - let implemented = map_roots(std::slice::from_ref(&root), &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &[entry], &arriving(), false) - .remove(0) - .at_ingestion - }; - assert_eq!( - at_ingestion(at_minute(60 * MINUTE)), - Err(IngestionPruneReason::RangeEvaluatedWholeWindow) - ); - assert_eq!(at_ingestion(at_minute(0)), Ok(())); - } - - /// A tumbling-window chain whose width does not divide a range-evaluated reader's - /// step is not offered: the candidate fails with the widths, and the - /// caller prunes it. 1-min tumbling windows serve 2-min steps. - #[test] - fn a_tumbling_window_chain_off_the_range_step_grid_is_rejected() { - use asap_types::workload::RangeEvaluation; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let stepping = |step_ms: u64| QueryWorkloadEntry { - range_evaluation: Some(RangeEvaluation { - step_ms, - span_ms: 60 * MINUTE, - }), - ..every_minute() - }; - let implement_for = |entry: QueryWorkloadEntry| { - optimize_physical_asap( - "L1", - std::slice::from_ref(&root), - &[entry], - &arriving(), - false, - ) - }; - let error = implement_for(stepping(30_000)).unwrap_err(); - assert!( - matches!( - error, - PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep { - tumbling_window_width_ms: MINUTE, - step_ms: 30_000, - } - ), - "{error}" - ); - assert_eq!( - error.to_string(), - "tumbling-window width 60000 ms does not divide the 30000 ms step of a range \ - evaluation that reads it" - ); - assert_eq!(implement_for(stepping(2 * MINUTE)).unwrap().len(), 2); - // A one-step range (span 0) is one evaluation per request, at the - // refresh: its step is not read. - let one_step = QueryWorkloadEntry { - range_evaluation: Some(RangeEvaluation { - step_ms: 30_000, - span_ms: 0, - }), - ..every_minute() - }; - assert_eq!(implement_for(one_step).unwrap().len(), 2); - } - - /// A build two queries read is also offered built once per evaluation - /// for both (`MatAtQuery`); a build one query reads is not. - #[test] - fn a_build_two_queries_read_may_be_built_once_per_evaluation() { - let QueryRoot::Operator(p99) = summary_root("quantile_over_time(0.99, m[1m])") else { - unreachable!() - }; - let build = OperatorNode::reachable(&p99) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .unwrap(); - let p50 = Rc::new( - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: Rc::clone(&build), - query: asap_types::ir::schema::SketchStatistic::Quantile { q: 0.5 }, - })) - .unwrap(), - ); - let roots = [QueryRoot::Operator(p99), QueryRoot::Operator(p50)]; - let candidates = - optimize_physical_asap("L1", &roots, &[], &DataWorkload::default(), false).unwrap(); - assert_eq!(candidates.len(), 2); - assert_eq!( - materialized(&candidates[1]), - [Materialization::MatAtQuery(Medium::Memory)] - ); - let [build] = &node_of(&candidates[1], "SummaryAgg")[..] else { - panic!("one shared build") - }; - assert_eq!( - build.materialization, - Materialization::MatAtQuery(Medium::Memory) - ); - } - - /// Two queries share one scan: one reads it through tumbling windows maintained at - /// ingestion time, the other at query time. The scan is split, one copy - /// per timing; with nothing materialized it stays one node. The shared - /// scan may also be built once per evaluation (`MatAtQuery`), but not - /// under ingestion-time tumbling windows, where only one query reads it at query - /// time. - #[test] - fn a_scan_two_queries_read_at_two_timings_is_split() { - let QueryRoot::Operator(tumbling_windows) = - in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")) - else { - unreachable!() - }; - let scan = OperatorNode::reachable(&tumbling_windows) - .into_iter() - .find(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .unwrap(); - let ranged = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(5 * MINUTE), - kind: asap_types::ir::TimeRangeKind::Range, - child: Rc::clone(&scan), - })) - .unwrap(); - let exact = asap_logical_asap_optimization::test_support::aggregate( - Reduction::by(vec![]), - vec![AggIntent::Max { col: None }], - vec![], - None, - ranged, - ); - let roots = [ - QueryRoot::Operator(tumbling_windows), - QueryRoot::Operator(exact), - ]; - let candidates = optimize_physical_asap( - "L1", - &roots, - &[every_minute(), every_minute()], - &arriving(), - false, - ) - .unwrap(); - assert_eq!(candidates.len(), 3); - let scans = |c: &PhysicalWorkloadPlan| { - let mut scans: Vec<_> = c - .dag - .nodes - .iter() - .filter(|n| matches!(n.payload, Payload::NonASAP(NonASAPOp::Scan { .. }))) - .map(|n| (n.output_state.timing, n.materialization)) - .collect(); - scans.sort_by_key(|(t, _)| *t == ExecutionTiming::QueryTime); - scans - }; - let not_materialized = Materialization::NotMaterialized; - assert_eq!( - scans(&candidates[0]), - [(ExecutionTiming::QueryTime, not_materialized)] - ); - assert_eq!( - scans(&candidates[1]), - [( - ExecutionTiming::QueryTime, - Materialization::MatAtQuery(Medium::Memory) - )] - ); - assert_eq!( - scans(&candidates[2]), - [ - (ExecutionTiming::IngestionTime, not_materialized), - (ExecutionTiming::QueryTime, not_materialized) - ] - ); - } - - /// Only the highest shared non-summary nodes are materialization - /// candidates, each with `NotMaterialized` and `MatAtQuery` only: a - /// 5 m range two queries read (not the scan below it, which only that - /// shared range reads for them), and the scan, which a third query - /// also reads through its own 1 m range. - #[test] - fn only_the_highest_shared_non_summary_nodes_are_candidates() { - let QueryRoot::Operator(p99) = summary_root("quantile_over_time(0.99, m[5m])") else { - unreachable!() - }; - let scan = OperatorNode::reachable(&p99) - .into_iter() - .find(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .unwrap(); - let ranged = |ms: u64| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(ms), - kind: asap_types::ir::TimeRangeKind::Range, - child: Rc::clone(&scan), - })) - .unwrap() - }; - let aggregate = |intent: AggIntent, input: Rc| { - QueryRoot::Operator(asap_logical_asap_optimization::test_support::aggregate( - Reduction::by(vec![]), - vec![intent], - vec![], - None, - input, - )) - }; - let five_minutes = ranged(5 * MINUTE); - let roots = [ - aggregate(AggIntent::Max { col: None }, Rc::clone(&five_minutes)), - aggregate(AggIntent::Min { col: None }, five_minutes), - aggregate(AggIntent::Max { col: None }, ranged(MINUTE)), - ]; - let mat_candidates = - materialization_candidates(&roots, &[], &DataWorkload::default(), false); - let found: Vec<_> = mat_candidates - .iter() - .map(|c| (c.node.operator.kind_name(), c.readers.clone(), c.options())) - .collect(); - let options = vec![ - Materialization::NotMaterialized, - Materialization::MatAtQuery(Medium::Memory), - ]; - assert_eq!( - found, - [ - ("TimeRange", vec![0, 1], options.clone()), - ("Scan", vec![0, 1, 2], options) - ] - ); - } - - /// Readers share one kept tumbling-window chain only when they evaluate at the - /// same instants: two readers every minute at phases 0 and 2 min do, - /// at phases 0 and 30 s they do not (each would need its own newest - /// tumbling window). A reader with no known phase is assumed to run with the - /// others. - #[test] - fn readers_at_different_phases_do_not_build_one_tumbling_window_per_evaluation() { - let at = |phase_ms: u64| { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(phase_ms), - }) - }; - let unknown_phase = every_minute().recurrence; - assert!(builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(0), &at(2 * MINUTE)] - )); - assert!(!builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(0), &at(30_000)] - )); - assert!(builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(30_000), &unknown_phase] - )); - assert!(!builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(0), &unknown_phase, &at(30_000)] - )); - } +mod catalog; - // A known evaluation phase is reduced modulo the width; a bare interval has - // none; a schedule has one only when all its times share it. - #[test] - fn tumbling_window_phase_of_a_repeated_demand() { - use asap_types::workload::{RepetitionInterval, TimestampMs}; - let at = |phase| RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(60_000), - evaluation_phase: TimestampMs(phase), - }; - let scheduled = |times: &[u64]| { - RepeatedDemand::Scheduled(times.iter().copied().map(TimestampMs).collect()) - }; - let phase = |demand| plan_tumbling_window_phase(&demand, 60_000); - assert_eq!(phase(at(56_000)), Some(56_000)); - assert_eq!(phase(at(176_000)), Some(56_000)); - assert_eq!( - phase(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))), - None - ); - assert_eq!(phase(scheduled(&[56_000, 176_000])), Some(56_000)); - assert_eq!(phase(scheduled(&[56_000, 150_000])), None); - } -} +pub use catalog::generate_physical_asap_memo_candidates; diff --git a/crates/physical-asap-optimization/src/split_by_timing.rs b/crates/physical-asap-optimization/src/split_by_timing.rs deleted file mode 100644 index 72dad6d63..000000000 --- a/crates/physical-asap-optimization/src/split_by_timing.rs +++ /dev/null @@ -1,375 +0,0 @@ -//! Splitting a shared sub-DAG by execution timing before Stage 2 applies a -//! materialization assignment ([`split_shared_by_timing`]). - -use std::collections::HashMap; -use std::rc::Rc; - -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{timing_under_consumer, MaterializationAssignment, OperatorNode}; - -/// Each (node, timing) pair's rebuilt node, with the original it copies. -type Copies = HashMap<(*const OperatorNode, ExecutionTiming), (Rc, Rc)>; - -/// Copy, for one consumer, every sub-DAG that `assignment` would reach with -/// two different timings from `roots` (one query each, or the roots of one -/// query), so that a workload whose CSE shared a `Scan` between an -/// ingestion-time summary and a query-time computation, in one query or -/// across queries, can still be timed. Only the conflicting sub-DAGs are -/// copied, one copy per timing; a sub-DAG reached with one timing stays one -/// `Rc`, shared by the roots that read it. Returns the (possibly rewritten) -/// roots, in order, and `assignment` remapped onto them: a rebuilt node -/// keeps its original's explicit materialization. -pub fn split_shared_by_timing( - roots: &[Rc], - assignment: &MaterializationAssignment, -) -> (Vec>, MaterializationAssignment) { - // First pass: the set of timings each node is reached with. - let mut reached: HashMap<*const OperatorNode, Vec> = HashMap::new(); - let mut forced = HashMap::new(); - fn collect( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - reached: &mut HashMap<*const OperatorNode, Vec>, - forced: &mut HashMap<*const OperatorNode, bool>, - ) { - let timing = timing_under_consumer(node, consumer, assignment, forced); - let entry = reached.entry(Rc::as_ptr(node)).or_default(); - if entry.contains(&timing) { - return; - } - entry.push(timing); - for child in node.children() { - collect(child, timing, assignment, reached, forced); - } - } - for root in roots { - collect( - root, - ExecutionTiming::QueryTime, - assignment, - &mut reached, - &mut forced, - ); - } - if reached.values().all(|timings| timings.len() <= 1) { - return (roots.to_vec(), assignment.clone()); - } - // Second pass: rebuild, giving each (node, timing) pair its own copy, - // kept with its original. - let mut copies: Copies = HashMap::new(); - fn rebuild( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - reached: &HashMap<*const OperatorNode, Vec>, - copies: &mut Copies, - forced: &mut HashMap<*const OperatorNode, bool>, - ) -> Rc { - let timing = timing_under_consumer(node, consumer, assignment, forced); - let key = (Rc::as_ptr(node), timing); - if let Some((_, done)) = copies.get(&key) { - return Rc::clone(done); - } - let conflicted = reached - .get(&Rc::as_ptr(node)) - .is_some_and(|timings| timings.len() > 1); - let mut changed = conflicted; - let operator = node.operator.map_children(|child| { - let rebuilt = rebuild(child, timing, assignment, reached, copies, forced); - changed |= !Rc::ptr_eq(&rebuilt, child); - rebuilt - }); - let out = if changed { - Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - Rc::clone(node) - }; - copies.insert(key, (Rc::clone(node), Rc::clone(&out))); - out - } - let split = roots - .iter() - .map(|root| { - rebuild( - root, - ExecutionTiming::QueryTime, - assignment, - &reached, - &mut copies, - &mut forced, - ) - }) - .collect(); - let mut remapped = assignment.clone(); - for (original, copy) in copies.values() { - if let Some(materialization) = assignment.get(original) { - remapped.set(copy, materialization); - } - } - (split, remapped) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::operator::agg_intent::AggIntent; - use asap_types::ir::operator::operator_properties::{Reduction, Source}; - use asap_types::ir::properties::{ExecutionDataState, ExecutionDataStateError}; - use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::ir::schema::sketch::{ - ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, - }; - use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; - use asap_types::ir::{ - apply_materialization_timings, ASAPOp, NodeToTimedNodeCache, NonASAPOp, Operator, - }; - use asap_types::physical::{Materialization, Medium}; - - const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - - fn scan() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("zone", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() - } - - fn kll() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) - } - - fn exact_sum() -> FieldDataType { - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - } - - fn agg(child: Rc, family: FieldDataType) -> Rc { - Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: family.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }), - Schema::lifted(vec![Field::new("state", family, false)], None), - ) - .with_guarantee(None), - ) - } - - fn estimate(child: Rc) -> Rc { - Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: child, - query: SketchStatistic::Quantile { q: 0.99 }, - }), - Schema::lifted( - vec![Field::plain("quantile_0_99", DataType::Float64, false)], - None, - ), - ) - .with_guarantee(None), - ) - } - - fn max(child: Rc) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Max { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child, - })) - .unwrap() - } - - fn concat(children: Vec>) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Concat { children })).unwrap() - } - - fn finalize(child: Rc) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child })) - .unwrap() - } - - fn child(node: &Rc) -> Rc { - Rc::clone(node.children()[0]) - } - - /// One shared sub-DAG reached as ingestion-time input and as query-time - /// input cannot be executed once for both; splitting it by timing first - /// makes the plan timeable. - #[test] - fn a_shared_subtree_reached_at_two_timings_conflicts() { - let shared = scan(); - let root = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![ - max(estimate(agg(Rc::clone(&shared), kll()))), - max(Rc::clone(&shared)), - ], - })) - .unwrap(); - assert_eq!( - apply_materialization_timings( - &root, - &MaterializationAssignment::with_default(AT_INGESTION), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::ConflictingTiming { - first: ExecutionDataState::INGESTION_ROWS, - second: ExecutionDataState::QUERY_ROWS, - }) - ); - let (split, assignment) = split_shared_by_timing( - std::slice::from_ref(&root), - &MaterializationAssignment::with_default(AT_INGESTION), - ); - assert!(apply_materialization_timings( - &split[0], - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .is_ok()); - } - - /// Splitting rebuilds the ancestors of a copied sub-DAG; an explicit - /// assignment on one of them follows it onto the rebuilt node. - #[test] - fn split_keeps_explicit_assignments() { - let shared = scan(); - let summary = agg(Rc::clone(&shared), kll()); - let root = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![max(estimate(Rc::clone(&summary))), max(Rc::clone(&shared))], - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, AT_INGESTION); - let (split, assignment) = split_shared_by_timing(std::slice::from_ref(&root), &assignment); - let timed = - apply_materialization_timings(&split[0], &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let summary = child(&child(&child(&timed))); - assert!(matches!( - summary.operator, - Operator::ASAP(ASAPOp::SummaryAgg { .. }) - )); - assert_eq!(summary.materialization, AT_INGESTION); - assert_eq!(summary.timing, Some(ExecutionTiming::IngestionTime)); - } - - /// Two queries share a scan: one reads it through a summary built at - /// ingestion time, the other at query time. Splitting the workload's - /// roots together copies the scan for one phase, so one timing pass - /// over both roots succeeds; a node both queries reach at one timing - /// stays one node. - #[test] - fn split_across_roots_copies_only_what_two_queries_read_at_two_timings() { - let shared = scan(); - let summary = agg(Rc::clone(&shared), kll()); - let by_summary = max(estimate(Rc::clone(&summary))); - let raw = max(Rc::clone(&shared)); - let common = max(estimate(agg(scan(), kll()))); - let roots = [ - concat(vec![Rc::clone(&by_summary), Rc::clone(&common)]), - concat(vec![raw, Rc::clone(&common)]), - ]; - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, AT_INGESTION); - let timed_unsplit = { - let mut cache = NodeToTimedNodeCache::new(); - roots - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache)) - .collect::, _>>() - }; - assert!(matches!( - timed_unsplit, - Err(ExecutionDataStateError::ConflictingTiming { .. }) - )); - let (split, assignment) = split_shared_by_timing(&roots, &assignment); - assert_eq!(split.len(), 2); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = split - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache).unwrap()) - .collect(); - let summary = child(&child(&child(&timed[0]))); - assert_eq!(summary.materialization, AT_INGESTION); - assert_eq!(child(&summary).timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!( - child(&child(&timed[1])).timing, - Some(ExecutionTiming::QueryTime) - ); - assert!(Rc::ptr_eq(timed[0].children()[1], timed[1].children()[1])); - } - - /// Splitting a workload that also shares a scan at two timings keeps a - /// conflicting assignment on a fixed-timing node: the split roots still - /// fail with the conflict instead of dropping the assignment. - #[test] - fn a_fixed_timing_conflict_survives_the_split() { - let shared = scan(); - let summary = agg(Rc::clone(&shared), kll()); - let pair = concat(vec![ - max(estimate(Rc::clone(&summary))), - max(Rc::clone(&shared)), - ]); - let state = agg(Rc::clone(&shared), exact_sum()); - let read = Rc::new( - (*finalize(Rc::clone(&state))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, AT_INGESTION); - assignment.set(&state, AT_INGESTION); - assignment.set(&read, AT_INGESTION); - let (split, assignment) = split_shared_by_timing(&[pair, read], &assignment); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Result, _> = split - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache)) - .collect(); - assert_eq!( - timed.err(), - Some( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: "FinalizeExactAccumulator", - fixed: ExecutionTiming::QueryTime, - materialization: AT_INGESTION, - } - ) - ); - } -} diff --git a/crates/plan-selection/Cargo.toml b/crates/plan-selection/Cargo.toml index ac64258a5..ab85b889e 100644 --- a/crates/plan-selection/Cargo.toml +++ b/crates/plan-selection/Cargo.toml @@ -9,33 +9,12 @@ edition = "2021" # the facade's `AssignmentRealizer`: no dependency on Stages 1 or 2, nor on # the executor (#1054 X1, #1047 P7). Tests build their inputs with them. [dependencies] -asap_sketchlib = { workspace = true } asap-types = { path = "../types" } asap-accuracy = { path = "../accuracy" } asap-cost = { path = "../cost" } thiserror = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" -rmp-serde = "1" good_lp = { version = "=1.15.3", default-features = false, features = ["highs"] } highs = "=2.4.0" - -[features] -# `reference_pricer`: the whole-workload selector over enumerated plans, for -# the developer tools' reference pricing (`asap-devtools`). -reference-pricer = [] -# `lp-diagnosis`: with `ASAP_STAGE3_LP_DIAGNOSIS=`, each selection MILP -# solve also reports where its LP relaxation loses its bound (measurement -# only, for the developer tools; a diagnosed run's timings are not -# comparable with an undiagnosed one's). -lp-diagnosis = [] - -[dev-dependencies] -asap-types = { path = "../types", features = ["test-support"] } -asap-accuracy = { path = "../accuracy", features = ["test-support"] } -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } -asap-frontend-promql = { path = "../frontend-promql" } -# The executor's charge for each summary state, checked against the bytes -# plan selection prices for it plus the accumulator's own fields (#947). diff --git a/crates/plan-selection/src/accuracy_binding.rs b/crates/plan-selection/src/accuracy_binding.rs deleted file mode 100644 index 853440ad0..000000000 --- a/crates/plan-selection/src/accuracy_binding.rs +++ /dev/null @@ -1,306 +0,0 @@ -//! Binding the statistics a root guarantee leaves symbolic: from the data -//! workload and the operand-domain contracts. - -use std::collections::HashMap; -use std::rc::Rc; - -use asap_accuracy::composition::{ - parse_raw_data_samples_sign_statistic, parse_source_norm_statistic, INPUT_ROW_COUNT, - PRODUCT_OPERAND_DOMAINS, STREAM_L1_NORM, -}; -use asap_accuracy::estimators::{hll, univmon}; -use asap_accuracy::propagation::{ - operand_domain_statistic, operand_guarantee, operand_of, project_column_guarantee, - root_guarantee, -}; -use asap_accuracy::OperandDomain; -use asap_types::ir::operator::operator_properties::BinaryOpKind; -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric}; -use asap_types::ir::scalar::ScalarValue; -use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::workload::DataWorkload; - -use crate::SelectionModels; -use asap_types::workload::bounds::declared_non_negative; - -/// [`root_guarantee`] with the statistics Stage 1 left symbolic bound: -/// - the input row count, `series` ([`series_read`](asap_types::workload::bounds::series_read) of `root`); -/// - each source's stream norm ([`source_norm_statistic`]), from the data -/// workload's per-source `stream_norms`; -/// - classic HLL's failure probability and the chance a UnivMon's top layer -/// is incomplete, from the data workload's per-source -/// `max_distinct_per_evaluation`; -/// - a product's or quotient's bound, from the deployment's enforced -/// operand domains ([`operand_domain_bounds`]). -/// -/// Every data-workload fact is declared: the caller drops facts that no -/// longer hold before planning. Any other statistic stays unknown, and an -/// unknown statistic never meets a target. -pub(crate) fn bound_root_guarantee( - root: &QueryRoot, - series: Option, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Option { - let domains = operand_domain_bounds(root, series, data, models); - let bind = |guarantee: &AccuracyGuarantee| { - bind_statistics(guarantee, &|statistic| { - workload_statistic(statistic, series, data) - .or_else(|| domains.get(statistic).copied().flatten()) - }) - }; - match root_guarantee(root) { - Some(guarantee) => Some(bind(&guarantee)), - None => asap_accuracy::propagation::bind_guarantees(root, &bind), - } -} - -/// `guarantee` with the statistics `value` supplies bound. A sign gate -/// ([`raw_data_samples_sign_statistic`](asap_accuracy::composition::raw_data_samples_sign_statistic)) -/// is an indicator that only 1 (proven) opens: any other value leaves it -/// unbound, so it can never read as a perfect bound. -pub(crate) fn bind_statistics( - guarantee: &AccuracyGuarantee, - value: &dyn Fn(&str) -> Option, -) -> AccuracyGuarantee { - guarantee.bind(&|statistic| { - let supplied = value(statistic); - match parse_raw_data_samples_sign_statistic(statistic) { - Some(_) => supplied.filter(|v| *v == 1.0), - None => supplied, - } - }) -} - -/// A data-workload statistic Stage 1 left symbolic: the input row count -/// (`series`, the series of the sources the query reads), one source's -/// stream norm, or classic HLL's failure probability or a UnivMon's -/// incomplete-top-layer probability given one source's distinct bound, -/// from source facts that hold at any planning time, or the sign of one -/// metric's samples, from its declared type. -pub(crate) fn workload_statistic( - statistic: &str, - series: Option, - data: &DataWorkload, -) -> Option { - // The sign of one metric's samples: proven (1) when the metric is - // declared a counter or declared non-negative; nothing is inferred from - // its name. - if let Some(metric) = parse_raw_data_samples_sign_statistic(statistic) { - return declared_non_negative(data, metric).then_some(1.0); - } - if statistic == INPUT_ROW_COUNT { - return series.map(|rows| rows as f64); - } - if let Some((precision, relative_error, source)) = - hll::parse_classic_failure_statistic(statistic) - { - let bound = data.sources.get(source)?.max_distinct_per_evaluation?; - return Some(hll::classic_failure_probability( - precision, - relative_error, - bound, - )); - } - if let Some((heap_size, layers, source)) = - univmon::parse_incomplete_top_layer_statistic(statistic) - { - let bound = data.sources.get(source)?.max_distinct_per_evaluation?; - return Some(univmon::incomplete_top_layer_probability( - heap_size, layers, bound, - )); - } - let (norm, source) = parse_source_norm_statistic(statistic)?; - let norms = data.sources.get(source)?.stream_norms?; - match norm { - STREAM_L1_NORM => norms.l1, - _ => norms.l2, - } -} - -/// One operand of a product or quotient, as Stage 3 finds its domain. -enum DomainOperand<'a> { - /// A planned operator: the deployment's `OperandDomain` for it. - Node(&'a Rc), - /// A literal: itself. - Literal(f64), - /// Anything else: no domain. - Other, -} - -impl<'a> DomainOperand<'a> { - fn of(expr: &'a ScalarExpr) -> Self { - match expr { - ScalarExpr::PromqlScalarFromVector(node) | ScalarExpr::ScalarSubquery(node) => { - Self::Node(node) - } - ScalarExpr::Literal(ScalarValue::Float64(value)) => Self::Literal(*value), - ScalarExpr::Literal(ScalarValue::Int64(value)) => Self::Literal(*value as f64), - _ => Self::Other, - } - } - - fn domain(&self, models: &SelectionModels<'_>) -> Option { - match self { - Self::Node(node) => models.evidence.operand_domain(node), - Self::Literal(value) => Some(OperandDomain { - lower: *value, - upper: *value, - }), - Self::Other => None, - } - } -} - -/// A product or quotient whose bound Stage 1 left symbolic on its operands' -/// domains ([`operand_domain_statistic`]). -struct DomainOperation<'a> { - statistic: &'static str, - operands: [(DomainOperand<'a>, AccuracyGuarantee); 2], -} - -/// Each operand-domain statistic in `root`, bound to the largest bound -/// among the operations that leave it symbolic, or `None` (unbound) when -/// any of them has no bound. Operations are found where the bound -/// originates (the same rule Stage 1 uses), in vector arithmetic, a scalar -/// root, and projection expressions. `series` is the root's input row -/// count ([`series_read`](asap_types::workload::bounds::series_read)). -fn operand_domain_bounds( - root: &QueryRoot, - series: Option, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> HashMap<&'static str, Option> { - let mut operations = Vec::new(); - if let QueryRoot::Scalar(expr) = root { - scalar_domain_operations(expr, &|_| None, &mut operations); - } - let nodes: Vec> = root - .operator_refs() - .iter() - .flat_map(|node| OperatorNode::reachable(node)) - .collect(); - for node in &nodes { - match node.non_asap() { - Some(NonASAPOp::BinaryOp { - operator: - BinaryOperator { - kind: BinaryOpKind::Arithmetic(op), - .. - }, - lhs, - rhs, - .. - }) => { - let (Some(l), Some(r)) = (operand_guarantee(lhs), operand_guarantee(rhs)) else { - continue; - }; - if let Some(statistic) = operand_domain_statistic(op, &l, &r) { - operations.push(DomainOperation { - statistic, - operands: [(DomainOperand::Node(lhs), l), (DomainOperand::Node(rhs), r)], - }); - } - } - Some(NonASAPOp::Project { cols, child, .. }) => { - for item in cols { - scalar_domain_operations( - &item.expr, - &|column| project_column_guarantee(child, column), - &mut operations, - ); - } - } - _ => {} - } - } - let mut bounds: HashMap<&'static str, Option> = HashMap::new(); - for operation in &operations { - let bound = domain_bound(operation, series, data, models); - let entry = bounds.entry(operation.statistic).or_insert(Some(0.0)); - *entry = entry.zip(bound).map(|(a, b)| a.max(b)); - } - bounds -} - -/// The operations in `expr` (and its sub-expressions) whose bound Stage 1 -/// left symbolic on operand domains. -fn scalar_domain_operations<'a>( - expr: &'a ScalarExpr, - column: &dyn Fn(usize) -> Option, - operations: &mut Vec>, -) { - if let ScalarExpr::Arithmetic { - op, left, right, .. - } = expr - { - if let (Some(l), Some(r)) = (operand_of(left, column), operand_of(right, column)) { - if let Some(statistic) = operand_domain_statistic(op, &l, &r) { - operations.push(DomainOperation { - statistic, - operands: [(DomainOperand::of(left), l), (DomainOperand::of(right), r)], - }); - } - } - } - for child in expr.children() { - scalar_domain_operations(child, column, operations); - } -} - -/// The bound of one product or quotient from its operands' domains: -/// - a product: `|x|·e_y + |y|·e_x + e_x·e_y`; -/// - a quotient: `(|x|·e_y + |y|·e_x) / (m·(m − e_y))` with `m` the least -/// `|y|`, which must exceed `e_y` (the denominator's domain is bounded -/// away from zero). -/// -/// `|x|`, `|y|` are the domains' largest magnitudes, and `e` each -/// operand's absolute error (a relative or cardinality bound times that -/// magnitude) after binding its data-workload statistics (`series`, the -/// root's input row count). `None` when an operand has no domain or an -/// unbound error. -fn domain_bound( - operation: &DomainOperation<'_>, - series: Option, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Option { - let [(x, gx), (y, gy)] = &operation.operands; - let (dx, dy) = (x.domain(models)?, y.domain(models)?); - let bind = - |g: &AccuracyGuarantee| g.bind(&|statistic| workload_statistic(statistic, series, data)); - let (gx, gy) = (bind(gx), bind(gy)); - let valid = - |d: &OperandDomain| d.lower.is_finite() && d.upper.is_finite() && d.lower <= d.upper; - if !(valid(&dx) && valid(&dy)) { - return None; - } - let magnitude = |d: &OperandDomain| d.lower.abs().max(d.upper.abs()); - let (mx, my) = (magnitude(&dx), magnitude(&dy)); - let error = |g: &AccuracyGuarantee, m: f64| { - let bound = g.spec.error.evaluate()?; - match g.spec.accuracy_metric { - _ if g.is_exact() => Some(0.0), - AccuracyMetric::ABSOLUTE => Some(bound), - AccuracyMetric::Relative => Some(bound * m), - _ => None, - } - }; - let (ex, ey) = (error(&gx, mx)?, error(&gy, my)?); - let bound = if operation.statistic == PRODUCT_OPERAND_DOMAINS { - mx * ey + my * ex + ex * ey - } else { - let least = if dy.lower > 0.0 { - dy.lower - } else if dy.upper < 0.0 { - -dy.upper - } else { - return None; - }; - if least <= ey { - return None; - } - (mx * ey + my * ex) / (least * (least - ey)) - }; - bound.is_finite().then_some(bound) -} diff --git a/crates/plan-selection/src/checks.rs b/crates/plan-selection/src/checks.rs deleted file mode 100644 index 3ac348d12..000000000 --- a/crates/plan-selection/src/checks.rs +++ /dev/null @@ -1,305 +0,0 @@ -//! Validity checks on one candidate: accuracy targets and latency. What -//! the executor or deployment cannot run (per-entity aggregates without a -//! kernel, query-time scans of raw data not retained) Stage 2 prunes. - -use asap_cost::{family_name, PlanCost, QueryLatency}; -use asap_types::ir::properties::{ - AccuracyGuarantee, AccuracyMetric, AccuracyScope, AccuracyTarget, -}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, WeightDomain}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use crate::accuracy_binding::bound_root_guarantee; -use crate::report::{AcceptedAccuracy, RejectionCause}; -use crate::SelectionModels; -use asap_types::workload::bounds::{confirmed_weights, series_read, values_per_result}; - -/// The first query whose latency per evaluation is over its -/// `max_latency_ms` bound or, with no bound broken, over its repeat -/// interval (a query evaluated every `I` ms whose evaluation takes longer -/// than `I` falls further behind with every evaluation), as a rejection -/// cause and reason. -pub(crate) fn latency_violation( - cost: &PlanCost, - entries: &[QueryWorkloadEntry], -) -> Option<(RejectionCause, String)> { - let per_query = || cost.usage.latency_per_query.iter().zip(entries).enumerate(); - let latency_text = |query: usize, latency: &QueryLatency| { - format!( - "q{}: latency {:.1} ms per evaluation (compaction {:.1} ms + query {:.1} ms)", - query + 1, - latency.total_ms(), - latency.compaction_ms, - latency.query_ms - ) - }; - let over_bound = - per_query().find_map( - |(query, (latency, entry))| match entry.requirements.max_latency_ms { - Some(bound) if latency.total_ms() > bound => Some(( - RejectionCause::LatencyBoundExceeded, - format!( - "{} is over its {bound} ms bound", - latency_text(query, latency) - ), - )), - _ => None, - }, - ); - over_bound.or_else(|| { - per_query().find_map(|(query, (latency, entry))| { - match entry.recurrence.fixed_interval_ms() { - Some(interval_ms) if latency.total_ms() > interval_ms as f64 => Some(( - RejectionCause::LatencyOverRepeatInterval, - format!( - "{} is over its {interval_ms} ms repeat interval, so its evaluations \ - cannot keep up", - latency_text(query, latency) - ), - )), - _ => None, - } - }) - }) -} - -/// Each entry's accuracy target, as the per-query targets Stage 3 checks. -pub(crate) fn accuracy_targets(entries: &[QueryWorkloadEntry]) -> Vec { - entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect() -} - -/// What Stage 3 accepted for each query ([`AcceptedAccuracy`]), or the -/// first query whose output is not shown to meet its target, as a reason. An output meets its target when its theoretical root guarantee -/// (composed in Stage 1 with the built-in model) satisfies it, or when the -/// deployment's empirical evidence shows that this candidate's output does -/// ([`AccuracyEvidenceProvider::output_guarantee`]). A -/// certified estimate inside the plan is not enough on its own: a difference -/// or ratio of certified estimates can be unbounded. The root guarantee's -/// symbolic statistics are bound first ([`bound_root_guarantee`]); one still -/// unknown leaves the target unmet. Count-Min is admitted only over update -/// weights proven non-negative, which its bound and the executor assume: an open -/// [`WeightDomain::RawDataSamplesOfMetric`] requirement is met when `data` declares -/// the metric a counter or non-negative ([`confirmed_weights`]), and unmet -/// otherwise. -/// -/// A target that sets no metric accepts a guarantee in any metric that meets -/// its ε, δ and scope ([`AccuracyModel::satisfies`]); one that sets a metric -/// accepts only guarantees in it (or convertible to it). -/// -/// The Count-Min weight check runs first and rejects as -/// [`RejectionCause::CountMinWeightsNotProvenNonNegative`]: the executor -/// cannot run such a plan. Then the deployment's accuracy model is asked -/// ([`AccuracyModel::accepts_output_regardless_of_guarantee`], through -/// `SelectionModels::accuracy`): an output it accepts is accepted -/// whatever its guarantee says, and the guarantee check's rejection, if any, -/// is only recorded ([`AcceptedAccuracy::guarantee_check_rejection`]). -/// Otherwise, an output with no guarantee from either -/// source (Stage 1 had no rule, and the evidence has none) is rejected as -/// [`RejectionCause::NoAccuracyGuarantee`], and any other miss as -/// [`RejectionCause::AccuracyTargetMissed`]. -pub(crate) fn accuracy_check( - candidate: &PhysicalWorkloadPlan, - targets: &[AccuracyTarget], - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Result, CheckFailure> { - let mut accepted = Vec::with_capacity(targets.len()); - for (query, (root, target)) in candidate.roots.iter().zip(targets).enumerate() { - let query = query + 1; - let mut sketches: Vec = Vec::new(); - for node in root - .operator_refs() - .iter() - .flat_map(|node| OperatorNode::reachable(node)) - { - let Operator::ASAP( - ASAPOp::SummaryAgg { family, input, .. } - | ASAPOp::SummaryTumblingWindowChain { family, input, .. }, - ) = &node.operator - else { - continue; - }; - if let FieldDataType::Sketch(kind, _) = family { - let name = format!("{:?}", kind.algorithm()); - if !sketches.contains(&name) { - sketches.push(name); - } - } - if matches!(family, FieldDataType::Sketch(kind, _) - if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap)) - && !matches!( - confirmed_weights(&input.weight_domain, data), - WeightDomain::NonNegative { .. } - ) - { - let unconfirmed = match &input.weight_domain { - WeightDomain::RawDataSamplesOfMetric { metric } => format!( - " (samples of `{metric}`, which is not declared a counter or \ - non-negative)" - ), - WeightDomain::CounterDerivativeOfMetric { metric } => format!( - " (rates or increases of `{metric}`, which is not declared a counter)" - ), - _ => String::new(), - }; - // Count-Min (plain, CmsWithHeap or Hydra Count-Min) over - // weights not proven non-negative does not run: the executor - // refuses CmsWithHeap and HydraCms when it builds the plan, - // and plain Count-Min adds a negative weight, which voids its - // never-underestimate bound. So - // this rejects the plan whatever the deployment's accuracy - // model accepts. - return Err(CheckFailure { - cause: RejectionCause::CountMinWeightsNotProvenNonNegative, - reason: format!( - "q{query}: {} needs non-negative update weights, and these are not \ - proven non-negative{unconfirmed}", - family_name(family) - ), - size_independent: true, - }); - } - } - // The guarantee check: the theoretical guarantee, then the - // deployment's evidence. - let checked = guarantee_check(query, root, target, data, models); - // The deployment's model may accept the output directly, whatever - // its guarantee says; the check's result is then only recorded. - if models - .accuracy - .accepts_output_regardless_of_guarantee(root, target) - { - accepted.push(AcceptedAccuracy { - target_metric: target.accuracy_metric, - accepted_metric: None, - accepted_regardless_of_guarantee: true, - guarantee_check_rejection: checked.err().map(|failure| failure.cause), - sketches, - }); - continue; - } - accepted.push(AcceptedAccuracy { - target_metric: target.accuracy_metric, - accepted_metric: checked?, - accepted_regardless_of_guarantee: false, - guarantee_check_rejection: None, - sketches, - }); - } - Ok(accepted) -} - -/// A check a candidate failed: why, with its numbers, and whether every -/// size of the same plan's summaries fails it too. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct CheckFailure { - pub(crate) cause: RejectionCause, - pub(crate) reason: String, - /// `true` when no other size of the plan's summaries can pass the check - /// (e.g. no rule composes a guarantee, or a bound stays unknown) and the - /// deployment's accuracy model and evidence cannot judge another size - /// otherwise; `false` when that is not proven. - pub(crate) size_independent: bool, -} - -/// Whether query `query`'s output `root` meets `target` by its theoretical -/// root guarantee or, failing that, the deployment's evidence -/// ([`accuracy_check`]): the metric of the guarantee that met it (`None` -/// for an exact output), or why not, as -/// [`RejectionCause::NoAccuracyGuarantee`] or -/// [`RejectionCause::AccuracyTargetMissed`]. -pub(crate) fn guarantee_check( - query: usize, - root: &QueryRoot, - target: &AccuracyTarget, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Result, CheckFailure> { - // A guarantee per value is stated over the values the target's scope - // covers: the whole result's `G`, or at most `⌊1 / share⌋` groups. - let series = series_read(root, data); - let values = values_per_result(root, series, data); - let over_scope = |g: AccuracyGuarantee| g.over_scope(target.accuracy_scope, values); - let guarantee = bound_root_guarantee(root, series, data, models).map(over_scope); - let meets = |g: &AccuracyGuarantee| models.accuracy.satisfies(g, target); - // The deployment's evidence is read only when the theoretical - // guarantee does not meet the target. - let evidence = || models.evidence.output_guarantee(root).map(over_scope); - let (met, evidence) = match guarantee.clone().filter(meets) { - Some(met) => (Some(met), None), - None => { - let evidence = evidence(); - (evidence.clone().filter(meets), evidence) - } - }; - if let Some(met) = met { - return Ok((!met.is_exact()).then_some(met.spec.accuracy_metric)); - } - // Whether the deployment's model or evidence may judge another size of - // the plan's summaries otherwise. - let model_or_evidence_may_judge = models.accuracy.may_accept_regardless_of_guarantee() - || models.evidence.may_provide_evidence(); - if guarantee.is_none() && evidence.is_none() { - return Err(CheckFailure { - cause: RejectionCause::NoAccuracyGuarantee, - reason: format!( - "q{query}: the output has no guarantee, and the deployment has no evidence \ - for it, so it cannot meet target {target:?}" - ), - // No rule composes one, whatever the sizes. - size_independent: !model_or_evidence_may_judge, - }); - } - // A bound or failure probability that stays unknown once bound is - // unknown at every size; a known one may shrink with larger summaries. - let unknown = |g: &AccuracyGuarantee| { - g.spec.error.evaluate().is_none() - || (target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none()) - }; - let size_independent = evidence.is_none() - && guarantee.as_ref().is_some_and(unknown) - && !model_or_evidence_may_judge; - Err(CheckFailure { - cause: RejectionCause::AccuracyTargetMissed, - size_independent, - reason: match guarantee { - Some(g) - if values.is_none() - && target.accuracy_scope == AccuracyScope::WholeResult - && target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none() => - { - format!( - "q{query}: the output's guarantee holds for each value, and the number \ - of values one evaluation returns is not bound (no declared series \ - count for a source a PromQL vector reads, and no declared \ - `distinct_values_per_label` for every group key nor a \ - `distinct_values_per_label_set` covering them, or a key that is not \ - an unchanged source column), so it does not meet \ - whole-result target {target:?}, and \ - the deployment has no evidence that it meets it (a target on the groups \ - holding a share of the records, `groups_with_record_share_at_least`, \ - covers a bound number of values)" - ) - } - Some(g) => format!( - "q{query}: the output guarantees a {:?} bound {:?}, failure probability \ - {:?}, which misses target {target:?}, and the deployment has no evidence \ - that it meets it", - g.spec.accuracy_metric, - g.spec.error.evaluate(), - g.spec.failure_probability.evaluate(), - ), - None => format!( - "q{query}: the output has no guarantee, and the deployment has no \ - evidence that it meets target {target:?}" - ), - }, - }) -} diff --git a/crates/plan-selection/src/groups.rs b/crates/plan-selection/src/groups.rs deleted file mode 100644 index 1421f5d2a..000000000 --- a/crates/plan-selection/src/groups.rs +++ /dev/null @@ -1,2163 +0,0 @@ -//! #1095 C3: what the owner-scoped candidate MILP (§2.0) is built from. Given -//! Stage 2's physical candidates (one per operator node, timing and -//! materialization; Stage 2 makes no copies), it computes: -//! -//! 1. **Candidate dominance** ([`prune_dominated_candidates`]): a candidate no optimal -//! plan under cost model 1 uses is deleted, with its links. -//! 2. **Per-candidate data** ([`CandidateData`]): whether the candidate outputs -//! approximate values and the budget parts it is sized for, and its usage -//! per execution, standing cost, held bytes and own latency, every number -//! from [`asap_cost::node_usage`]. -//! 3. **Owners** ([`OwnerRegion`]): one per query and one per possible -//! instance of each shareable candidate, each with its region of stateless -//! candidates, the links inside it and the shareable candidates it may read. -//! 4. **Accuracy weights** ([`AccuracyWeight`]) per query and approximate -//! candidate, and pairs excluded together, from the composition of the -//! query's outputs. -//! -//! Nothing calls it yet; the MILP (C4) will. -//! -//! ## Usage per execution and its context -//! -//! A candidate's usage is [`asap_cost::node_usage`] of its operator node at one -//! execution per second (`rate = (1, 1.0)`); its standing cost is the same -//! call's cost at no execution (`rate = (1, 0.0)`), and its execution cost -//! the difference. `node_usage` also needs facts the candidate alone does not -//! fix, its *context*: -//! - the outputs of the producers it reads (one per input; the producer -//! candidates of an input may output different statistics); -//! - for a query-time scan, the window each reader reads, through time -//! shifts ([`window_read_through`], which follows -//! [`PhysicalASAPDAG::scan_window`] for a plan with that one reader); -//! - for a pane chain kept beyond one evaluation, the panes it keeps, which -//! grow with the range-evaluation span of the queries reaching it -//! ([`asap_cost::tumbling_window_retention_ms`], [`asap_cost::retained_tumbling_windows`]). -//! -//! Every combination of these options is costed; a combination -//! `asap-cost` cannot cost is skipped -//! (no plan can use it). When the numbers agree, the usage is -//! plan-independent; otherwise [`CandidateData::usage_depends_on`] says which -//! options changed it, and [`CandidateData::usage`] holds the first costed -//! combination's numbers. An ingestion-time node's maintained window is -//! read from the first producer candidate of each input -//! ([`asap_cost::maintained_window_ms`]). -//! -//! Candidate links can close a cycle (on #1036 workload-01, a widened scan's -//! `Project` that reads its own memo group). Candidates are costed producers first; on a -//! cycle, a candidate is costed from its producers already costed and marked -//! [`UsageContext::ProducerOnACycle`]. - -//! ## Dominance (exact under cost model 1) -//! -//! Candidate `k` is deleted when another candidate `k'` (the *dominating* candidate): -//! - is in the same memo group, at the same timing and materialization, -//! with the same held state; -//! - has the same producer candidates per input, the same readers (every -//! (reader candidate, input) that may read `k` may read `k'`, and nothing -//! else), the same output references it may produce, the same reader -//! constraints and keep flags; -//! - has the same `(approx, parts)`; -//! - has a guarantee at least as strong ([`guarantee_at_least_as_strong`]); -//! - outputs the same schema, sources, state family and unchanged columns -//! with an operator of the same kind (pane chains: the same panes), so its -//! readers are costed with the same formulas; -//! - and has every number at most `k`'s: CPU operations and scanned bytes -//! per execution, latency (work per execution), output rows and bytes, -//! kept memory (average and peak), disk bytes, held bytes and raw data -//! kept. -//! -//! Both candidates' usage must be plan-independent (otherwise the comparison -//! would hold for one context only). Readers' usage is assumed -//! non-decreasing in their inputs' rows and bytes. Among candidates that -//! dominate each other the one with the smallest id is kept. Every link to -//! a deleted candidate is deleted; its readers still read the dominating candidate. -//! -//! ## Accuracy weights -//! -//! `w(q, k)` says whether query `q` may read approximate candidate `k` (a -//! sketch [`ASAPOp::SummaryAgg`] or [`ASAPOp::SummaryTumblingWindowChain`], what -//! [`asap_accuracy::propagation::approximate_inputs`] counts), and pairs of -//! approximate candidates `q` may not read together. Both come from the -//! composition itself, never from an operator table -//! ([`accuracy_positions`]): `q`'s outputs are built over the candidates -//! with `k` approximate and every other candidate exact, their guarantees -//! composed by `asap-accuracy` ([`asap_accuracy::propagation::composed_guarantee`]) -//! and checked by the same check the selected plan passes -//! ([`crate::checks::guarantee_check`], with the deployment's model): -//! - every output exact: `k`'s values do not reach the output (only a row -//! count does), and `k` gets no weight; -//! - every output meets the target: `w(q, k) = 1`, one share of `q`'s -//! budget ([`asap_accuracy::budget`]); `k`'s sizes are those of its -//! `parts`, so a lone denominator sized for the whole target -//! (`ε/(1 − ε) > ε`) fails here; -//! - otherwise [`AccuracyWeight::Unusable`], with the check's cause -//! ([`UnusableReason`]): e.g. a rank error through any arithmetic, or an -//! HLL whose declared distinct bound is outside its contract. -//! -//! For each pair `k1, k2` of candidates usable alone, both sized for at -//! least two shares, whose values meet in one plan of `q` (under two inputs -//! of one candidate, under two references of a scalar root, or one below -//! the other), the outputs are built and checked with both approximate; a -//! pair that fails is excluded (`a[q, k1] + a[q, k2] ≤ 1`). A pair with a -//! member sized for one share is excluded by the shares row already. The -//! rows still bound the shares a plan reads by each candidate's `parts`, -//! which the equal split proves for one product or quotient of two inputs; -//! three or more approximate inputs are not built together. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; - -use asap_cost::{CostInputs, NodeUsage, ProducerOutput}; -use asap_types::ir::materialized_view::RefreshWindow; -use asap_types::ir::physical_export::{ - PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, ScanWindow, -}; -use asap_types::ir::properties::execution_data_state::{DataPrimitive, ExecutionDataState}; -use asap_types::ir::properties::{AccuracyGuarantee, ExecutionTiming}; -use asap_types::ir::schema::{FieldDataType, Schema}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use asap_types::physical::Materialization; -use asap_types::stages::{ - output_references, MemoGroupId, OperatorNodeOfCandidateForMemoGroupId, - PhysicalASAPMemoCandidates, -}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use crate::SelectionModels; - -use asap_types::stages::{PhysicalCandidateId, ReaderConstraint}; - -/// What the MILP is built from (module documentation). -#[derive(Debug, Clone)] -pub(crate) struct MilpInputs { - /// Indexed by [`PhysicalCandidateId`]: each candidate's data, pruned or not. - pub(crate) candidate_data: Vec, - /// The candidates dominance deleted, ascending. - pub(crate) pruned_candidates: Vec, - /// Indexed by [`PhysicalCandidateId`]: per input, the producer candidates left - /// after pruning (empty for a pruned candidate). - pub(crate) producer_candidates_per_input: Vec>>, - /// Per output reference (`output_references` of the query roots): the - /// candidates left that may produce it, `Out(r)`, ascending. - pub(crate) output_candidates: Vec>, - /// Indexed by [`PhysicalCandidateId`]: where it runs. - pub(crate) kind: Vec, - /// Indexed by [`PhysicalCandidateId`]: the result it gives. - pub(crate) slot: Vec, - /// The execution contexts: one per query (`0..Q`, in workload order), - /// then the ingestion pipelines `I_r` some context reads, ascending by - /// [`IngestionPipeline`], then one per shareable candidate some context - /// may read, ascending by candidate. - pub(crate) contexts: Vec, - /// Per shareable candidate with a context, its index in `contexts`. - pub(crate) context_of_shareable: BTreeMap, - /// Per ingestion-time shareable candidate whose context reads a - /// candidate of an `I_r`: the index of that `I_r` in `contexts`. - pub(crate) ingestion_context_of_shareable: BTreeMap, - /// Per query: `w(q, k)` for each approximate candidate `k` whose values - /// reach its output. - pub(crate) accuracy_weights: Vec>, - /// Per query: the pairs `(k1, k2)`, `k1 < k2`, of candidates usable alone - /// that it may not read together. - pub(crate) accuracy_excluded_pairs: Vec>, -} - -/// One candidate's data for the MILP. -#[derive(Debug, Clone)] -pub(crate) struct CandidateData { - /// Whether it outputs approximate values: a sketch `SummaryAgg` or - /// `SummaryTumblingWindowChain`, each an approximate input of the queries reading it. - pub(crate) approximate: bool, - /// `parts(k)`: its operator node's `accuracy_budget_parts` (Stage 2's - /// `OperatorNodeOfCandidateForMemoGroup`: for a factored node, the - /// largest among its candidates). - pub(crate) accuracy_budget_parts: usize, - /// Its usage under the first context, or why `asap-cost` could not cost - /// it. - pub(crate) usage: Result, - /// The context options that changed its usage, ascending; empty when it - /// is plan-independent. - pub(crate) usage_depends_on: Vec, - /// The inputs whose producer (or that producer's output) changes its - /// usage or its own output, ascending. - pub(crate) inputs_changing_usage: Vec, - /// Its usage and its own output (an index below `output_count`) per - /// choice of a producer and that producer's output for each of - /// `inputs_changing_usage` (in that order); `None` when `asap-cost` could - /// not cost that choice, so no plan can make it. Empty when - /// `inputs_changing_usage` is. - pub(crate) usage_per_producers: Vec<(Vec, Option)>, - /// For a query-time scan whose usage depends on the window its reader - /// reads ([`UsageContext::ScanWindowOfReader`]): its usage per window it - /// can be costed at, with its output there. Empty otherwise. - pub(crate) usage_per_window: Vec<(ScanWindow, CandidateUsage, usize)>, - /// How many distinct outputs (statistics its readers are costed from) - /// it can give, by context. - pub(crate) output_count: usize, - /// For a query-time scan: per reader, the windows it may read the scan - /// at (through time shifts, [`window_read_through`]). Empty for any - /// other candidate. - pub(crate) windows_per_reader: Vec<(PhysicalCandidateId, Vec)>, - /// For a kept pane chain whose usage depends on the panes it retains - /// ([`UsageContext::PaneRetentionOfReachingQueries`]): its usage per - /// retained pane count. Empty otherwise. - pub(crate) usage_per_retained_panes: Vec<(u64, CandidateUsage)>, - /// With `usage_per_retained_panes`: per query that can reach it, the - /// panes it needs retained. - pub(crate) retained_panes_per_query: Vec<(usize, u64)>, - /// Whether some entry of `usage_per_producers` stands for several usages - /// (a producer whose own output depends on its context), of which the - /// costliest is kept. - pub(crate) producer_usage_ambiguous: bool, -} - -impl Default for CandidateData { - fn default() -> Self { - Self { - approximate: false, - accuracy_budget_parts: 1, - usage: Err("not costed".to_string()), - usage_depends_on: Vec::new(), - inputs_changing_usage: Vec::new(), - usage_per_producers: Vec::new(), - usage_per_window: Vec::new(), - output_count: 1, - windows_per_reader: Vec::new(), - usage_per_retained_panes: Vec::new(), - retained_panes_per_query: Vec::new(), - producer_usage_ambiguous: false, - } - } -} - -/// One candidate's usage, from [`asap_cost::node_usage`] (module -/// documentation). -#[derive(Debug, Clone, Copy, PartialEq)] -pub(crate) struct CandidateUsage { - /// `u(k)`: CPU operations of one execution. - pub(crate) cpu_ops_per_execution: f64, - /// `u(k)`: bytes one execution scans. - pub(crate) scanned_bytes_per_execution: f64, - /// Milliseconds of one execution on one core. - pub(crate) work_ms_per_execution: f64, - /// `d_k`: what it adds to a reaching query's latency in - /// [`asap_cost::plan_cost`], its work per execution at query time; 0 - /// at ingestion time, whose latency is the plan-level compaction term. - pub(crate) own_latency_ms: f64, - /// Cost model 1: the cost of one execution; `None` under other models. - pub(crate) cost_per_execution: Option, - /// Cost model 1: the cost per second of what it keeps whatever runs it - /// (kept memory and disk); `None` under other models. - pub(crate) standing_cost_per_second: Option, - /// Memory kept between executions, averaged over time. - pub(crate) average_memory_bytes: f64, - pub(crate) peak_memory_bytes: u64, - pub(crate) disk_bytes: u64, - /// Query-time memory held while an evaluation reading it runs (a - /// `MatAtQuery` state, or a not-materialized node's working memory). - pub(crate) held_bytes: u64, - /// Raw data a query-time scan keeps on disk. - pub(crate) raw_data_kept_bytes: u64, - /// An in-memory ingestion-time state's bytes per window. - pub(crate) maintained_window_bytes: u64, - pub(crate) output_rows: u64, - pub(crate) output_bytes: u64, -} - -/// A candidate's usage and its output (an index below -/// [`CandidateData::output_count`]) in one context. -pub(crate) type UsageAndOutput = (CandidateUsage, usize); - -/// One producer choice for an input: a producer candidate, and for one -/// with several outputs ([`CandidateData::output_count`]), which output. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct ProducerChoice { - pub(crate) class: PhysicalCandidateId, - pub(crate) output: Option, -} - -/// A context option that changed a candidate's usage. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) enum UsageContext { - /// Which producer candidate (of different output) an input reads. - ProducerOutput, - /// Which reader reads a query-time scan, so which window it scans. - ScanWindowOfReader, - /// Which queries reach a kept pane chain, so how many panes it keeps. - PaneRetentionOfReachingQueries, - /// A producer candidate on a cycle of candidate links was not costed yet, so - /// its outputs were not among the options. - ProducerOnACycle, - /// `asap-cost` could not cost some contexts (e.g. a scan read with no - /// window from a source without `rows_held`); the usage is the first - /// costed one's. - UncostableContext, -} - -/// A candidate dominance deleted. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct DominatedCandidate { - pub(crate) class: PhysicalCandidateId, - pub(crate) dominated_by: PhysicalCandidateId, - pub(crate) reason: String, -} - -/// `w(q, k)`: one share of `q`'s budget, or unusable. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) enum AccuracyWeight { - One, - /// `∞`: `k` is unusable for `q`. - Unusable(UnusableReason), -} - -/// Why a position's weight is `∞`: the check of `q`'s outputs with `k` -/// approximate failed. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) enum UnusableReason { - /// Some output has no guarantee - /// ([`RejectionCause::NoAccuracyGuarantee`](crate::report::RejectionCause::NoAccuracyGuarantee)). - NoAccuracyGuarantee, - /// Some output's guarantee misses the target - /// ([`RejectionCause::AccuracyTargetMissed`](crate::report::RejectionCause::AccuracyTargetMissed)). - AccuracyTargetMissed, - /// Not checked, so not usable: a node that does not rebuild over its - /// producers. - NotChecked, -} - -/// Where a candidate runs (#1095 §2.0, "Memo groups and execution -/// contexts"), by its materialization alone, as `asap_cost::plan_cost` -/// counts its executions: -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub(crate) enum RunKind { - /// `NotMaterialized` at query time: computed once in each execution - /// context (a query evaluation, or a shared query-time candidate's - /// inputs) that selects it. - PerContext, - /// `NotMaterialized` at ingestion time: runs in the ingestion pipeline - /// `I_r` of its readers' refresh window `r`, once per second of - /// arriving data for all of them. - Ingestion, - /// Materialized (`MatAtIngestion`, `MatAtQuery`, - /// `MatAtQueryKeptAcrossEvaluations`): selected once, its own execution - /// context for its inputs, charged once. - Shareable, -} - -/// The [`RunKind`] of a candidate at `timing` and `materialization`. -pub(crate) fn run_kind(timing: ExecutionTiming, materialization: Materialization) -> RunKind { - match (materialization, timing) { - (Materialization::NotMaterialized, ExecutionTiming::QueryTime) => RunKind::PerContext, - (Materialization::NotMaterialized, ExecutionTiming::IngestionTime) => RunKind::Ingestion, - _ => RunKind::Shareable, - } -} - -/// What "one candidate per memo group per context" (row 2) ranges over: -/// the candidates that give one result. A memo group's implementation -/// roots give the group's result; an implementation's private nodes -/// (`is_implementation_root == false`) give an intermediate result of -/// their own, so each private operator node is its own slot (its -/// candidates differ only in timing and materialization). -/// -/// The memo group graph is not acyclic: a widened scan puts a `Project` in -/// the memo group of the scan it reads, so a group's candidate may read -/// its own group. Such a candidate and its producer cannot share one slot -/// (one context would need both), so a group's roots are split by -/// *self-read depth*: 0 for a root reading no root of its own group (the -/// scan), 1 for one that may (the `Project`). A depth-1 root reading -/// another depth-1 root of its group in one context (a `Project` over a -/// `Project` of the widened scan) is then a read of its own slot, which -/// [`CandidateLinks`] drops for candidates that run in a context. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum ResultSlot { - Group(MemoGroupId, u32), - PrivateNode(OperatorNodeOfCandidateForMemoGroupId), -} - -/// The [`ResultSlot`] of every candidate (see there). -pub(crate) fn result_slots(physical: &PhysicalASAPMemoCandidates) -> Vec { - let count = physical.candidates.len(); - let root = |candidate: usize| { - let operator_node = physical.candidates[candidate].operator_node; - let node = &physical.operator_nodes_of_candidates_for_memo_groups[operator_node.0]; - node.is_implementation_root - .then(|| node.group()) - .ok_or(operator_node) - }; - // Self-read depth: 1 for a root that may read a root of its own group. - let depth: Vec = (0..count) - .map(|candidate| { - let Ok(group) = root(candidate) else { - return 0; - }; - let reads_own_group = physical.candidates[candidate] - .producer_candidates_per_input - .iter() - .flat_map(|producers| producers.iter()) - .any(|producer| producer.0 != candidate && root(producer.0) == Ok(group)); - u32::from(reads_own_group) - }) - .collect(); - (0..count) - .map(|candidate| match root(candidate) { - Ok(group) => ResultSlot::Group(group, depth[candidate]), - Err(operator_node) => ResultSlot::PrivateNode(operator_node), - }) - .collect() -} - -/// Per candidate of a [`PhysicalASAPMemoCandidates`]: its producer -/// candidates per input, ascending and without repeats, and its -/// [`RunKind`]. -struct CandidateLinks { - producers_per_input: Vec>>, - kind: Vec, -} - -impl CandidateLinks { - fn new(physical: &PhysicalASAPMemoCandidates) -> Self { - let producers_per_input: Vec>> = physical - .candidates - .iter() - .map(|candidate| { - candidate - .producer_candidates_per_input - .iter() - .map(|producers| { - let set: BTreeSet<_> = producers.iter().copied().collect(); - set.into_iter().collect() - }) - .collect() - }) - .collect(); - let kind: Vec = physical - .candidates - .iter() - .map(|candidate| run_kind(candidate.timing, candidate.materialization)) - .collect(); - // With one candidate per result slot per context, a candidate that - // runs in a context cannot read its own slot there (only a self-loop - // could): such links are dropped (a widened scan's `Project` lists - // its own memo group). A shareable candidate reads in its own - // context, so it keeps them, but never reads itself. - let slot = result_slots(physical); - let producers_per_input = producers_per_input - .into_iter() - .enumerate() - .map(|(reader, inputs): (usize, Vec>)| { - inputs - .into_iter() - .map(|producers| { - producers - .into_iter() - .filter(|producer| { - producer.0 != reader - && (kind[reader] == RunKind::Shareable - || slot[producer.0] != slot[reader]) - }) - .collect() - }) - .collect() - }) - .collect(); - Self { - producers_per_input, - kind, - } - } - - /// Whether `candidate` is materialized: selected once, its own context. - fn is_shareable(&self, candidate: PhysicalCandidateId) -> bool { - self.kind[candidate.0] == RunKind::Shareable - } -} - -/// Per candidate, its reader constraints, each as an id equal for equal -/// constraints once the constrained candidate is left out, ascending and -/// without repeats. -fn reader_constraint_ids_per_candidate(physical: &PhysicalASAPMemoCandidates) -> Vec> { - let mut ids: HashMap = HashMap::new(); - let mut per_candidate = vec![Vec::new(); physical.candidates.len()]; - for constraint in &physical.reader_constraints { - let mut unowned = constraint.clone(); - *unowned.candidate_mut() = PhysicalCandidateId(usize::MAX); - let next = ids.len() as u32; - per_candidate[constraint.candidate().0].push(*ids.entry(unowned).or_insert(next)); - } - for constraint_ids in &mut per_candidate { - constraint_ids.sort_unstable(); - constraint_ids.dedup(); - } - per_candidate -} - -/// [`MilpInputs`] for `physical`'s candidates; `entries[q]` is query `q`'s -/// workload entry. Accuracy is checked with the built-in model and no -/// evidence ([`prepare_milp_inputs_with`]). -#[cfg(test)] -pub(crate) fn prepare_milp_inputs( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - cost_inputs: &CostInputs, -) -> MilpInputs { - let models = SelectionModels { - accuracy: &asap_accuracy::BuiltinAccuracyModel, - evidence: &asap_accuracy::NoAccuracyEvidence, - cost_inputs, - }; - prepare_milp_inputs_with(physical, entries, data, &models) -} - -/// [`MilpInputs`] for `physical`'s candidates, its accuracy positions -/// checked with `models` (the deployment's accuracy model and evidence) and -/// its usage costed with `models.cost_inputs`. -pub(crate) fn prepare_milp_inputs_with( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> MilpInputs { - let cost_inputs = models.cost_inputs; - let links = &CandidateLinks::new(physical); - let class_count = physical.candidates.len(); - let all_alive = vec![true; class_count]; - let unpruned_links = producer_candidates_left(links, &all_alive); - let unpruned_outputs = output_candidates(physical, &all_alive); - let unpruned_reach = queries_reaching(physical, &unpruned_links, &unpruned_outputs); - let refresh_windows = refresh_windows(physical, links); - // Every context before pruning, for the windows each scan may be read - // at within one context. - let all_contexts = execution_contexts( - physical, - &links.kind, - &unpruned_links, - &unpruned_outputs, - &refresh_windows, - |_| true, - ) - .contexts; - let candidate_data = candidate_data( - physical, - links, - entries, - &unpruned_reach, - &all_contexts, - &unpruned_outputs, - data, - cost_inputs, - ); - let pruned_candidates = prune_dominated_candidates( - &dominance_profiles(physical, links, &unpruned_outputs), - &candidate_data, - ); - let mut alive = all_alive; - for pruned in &pruned_candidates { - alive[pruned.class.0] = false; - } - let producer_candidates_per_input = producer_candidates_left(links, &alive); - let output_candidates = output_candidates(physical, &alive); - let kind = links.kind.clone(); - let slot = result_slots(physical); - let usable = |class: PhysicalCandidateId| candidate_data[class.0].usage.is_ok(); - let ExecutionContexts { - contexts, - context_of_shareable, - ingestion_context_of_shareable, - } = execution_contexts( - physical, - &kind, - &producer_candidates_per_input, - &output_candidates, - &refresh_windows, - usable, - ); - let AccuracyPositions { - weights: accuracy_weights, - excluded_pairs: accuracy_excluded_pairs, - } = accuracy_positions::accuracy_positions( - physical, - &candidate_data, - &producer_candidates_per_input, - &output_candidates, - entries, - data, - models, - ); - MilpInputs { - candidate_data, - pruned_candidates, - producer_candidates_per_input, - output_candidates, - kind, - slot, - contexts, - context_of_shareable, - ingestion_context_of_shareable, - accuracy_weights, - accuracy_excluded_pairs, - } -} - -/// Per candidate, per input, its producer candidates that are `alive` (none for a -/// candidate that is not). -fn producer_candidates_left( - links: &CandidateLinks, - alive: &[bool], -) -> Vec>> { - links - .producers_per_input - .iter() - .enumerate() - .map(|(class, producers_per_input)| match alive[class] { - false => Vec::new(), - true => producers_per_input - .iter() - .map(|producers| { - producers - .iter() - .copied() - .filter(|producer| alive[producer.0]) - .collect() - }) - .collect(), - }) - .collect() -} - -/// `Out(r)` per output reference: the `alive` candidates among its output -/// candidates, ascending. -fn output_candidates( - physical: &PhysicalASAPMemoCandidates, - alive: &[bool], -) -> Vec> { - (0..physical.output_reference_count()) - .map(|reference| { - let set: BTreeSet<_> = physical - .query_output_candidates(reference) - .filter(|candidate| alive[candidate.0]) - .collect(); - set.into_iter().collect() - }) - .collect() -} - -/// Per candidate, the queries whose output candidates reach it through the links -/// `producer_candidates_per_input`, ascending. -fn queries_reaching( - physical: &PhysicalASAPMemoCandidates, - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], -) -> Vec> { - let mut reaching = vec![Vec::new(); producer_candidates_per_input.len()]; - let output_refs = output_references(&physical.query_roots); - for query in 0..physical.query_roots.len() { - let mut seen = vec![false; producer_candidates_per_input.len()]; - let mut stack: Vec = output_refs - .iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .flat_map(|(reference, _)| output_candidates[reference].iter().copied()) - .collect(); - while let Some(class) = stack.pop() { - if std::mem::replace(&mut seen[class.0], true) { - continue; - } - reaching[class.0].push(query); - stack.extend( - producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied(), - ); - } - } - reaching -} - -/// Which execution context (#1095 §2.0): a query evaluation, an ingestion -/// pipeline `I_r`, or a shareable candidate's own inputs. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum ContextKind { - Query(usize), - Ingestion(IngestionPipeline), - Shared(PhysicalCandidateId), -} - -/// Which ingestion pipeline `I_r` (#1095 §2.5 O3(b)): the deployment -/// refreshes the ingestion-time views of one refresh window in one run, so -/// `asap_cost::execution_demand::computations` charges a `NotMaterialized` -/// ingestion-time node once per refresh window of the views reading it. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum IngestionPipeline { - /// The views of this refresh window. - Window(RefreshWindow), - /// An ingestion-time shareable candidate whose window is unbounded or - /// empty: refreshed alone. - Alone(PhysicalCandidateId), -} - -/// The [`IngestionPipeline`] of ingestion-time shareable candidate -/// `shareable`, from its refresh window ([`refresh_windows`]). -pub(crate) fn ingestion_pipeline( - shareable: PhysicalCandidateId, - refresh_windows: &HashMap>, -) -> IngestionPipeline { - match refresh_windows.get(&shareable) { - Some(Some(window)) => IngestionPipeline::Window(*window), - _ => IngestionPipeline::Alone(shareable), - } -} - -/// Each ingestion-time shareable candidate's refresh window, as -/// `asap_cost::execution_demand::computations` computes it on the realized -/// plan ([`PhysicalASAPDAG::refresh_windows`]): on the DAG of the candidates -/// below it, each input read from its first producer candidate. That is -/// exact when the producer candidates of each input read one window, which -/// is debug-asserted ([`producer_window_disagreements`]). -fn refresh_windows( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, -) -> HashMap> { - let views: Vec = (0..physical.candidates.len()) - .map(PhysicalCandidateId) - .filter(|&class| { - links.is_shareable(class) - && physical.candidates[class.0].timing == ExecutionTiming::IngestionTime - }) - .collect(); - let dag = dag_below(physical, links, &views); - debug_assert!( - producer_window_disagreements(links, &dag).is_empty(), - "producer candidates of one input read different windows: {:?}", - producer_window_disagreements(links, &dag) - ); - dag.refresh_windows(views.iter().map(|view| view.0)) - .into_iter() - .map(|(id, window)| (PhysicalCandidateId(id), window)) - .collect() -} - -/// The DAG of `tops` and every candidate below them, without edges: each -/// candidate a node ([`catalog_node`]) whose inputs are its first producer -/// candidates. -fn dag_below( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, - tops: &[PhysicalCandidateId], -) -> PhysicalASAPDAG { - let mut nodes = Vec::new(); - let mut seen = HashSet::new(); - let mut stack = tops.to_vec(); - while let Some(class) = stack.pop() { - if seen.insert(class) { - nodes.push(catalog_node(physical, links, class)); - stack.extend(links.producers_per_input[class.0].iter().flatten()); - } - } - let nodes: Vec<&PhysicalASAPDAGNode> = nodes.iter().collect(); - nodes_dag(&nodes) -} - -/// The `(candidate, input)` pairs of `dag`'s candidates -/// ([`dag_below`]) whose producer candidates have different view windows -/// ([`PhysicalASAPDAG::view_windows`]). -fn producer_window_disagreements( - links: &CandidateLinks, - dag: &PhysicalASAPDAG, -) -> Vec<(PhysicalCandidateId, usize)> { - let windows = dag.view_windows(dag.nodes.iter().map(|node| node.id)); - let mut disagreements = Vec::new(); - for node in &dag.nodes { - for (input, producers) in links.producers_per_input[node.id].iter().enumerate() { - let mut read = producers.iter().map(|producer| windows[&producer.0]); - if let Some(first) = read.next() { - if read.any(|window| window != first) { - disagreements.push((PhysicalCandidateId(node.id), input)); - } - } - } - } - disagreements -} - -/// [`producer_window_disagreements`] over every candidate of `physical`. -#[cfg(test)] -pub(crate) fn producer_window_disagreements_of_all( - physical: &PhysicalASAPMemoCandidates, -) -> Vec<(PhysicalCandidateId, usize)> { - let links = CandidateLinks::new(physical); - let all: Vec = (0..physical.candidates.len()) - .map(PhysicalCandidateId) - .collect(); - producer_window_disagreements(&links, &dag_below(physical, &links, &all)) -} - -/// One execution context and the candidates that may give a result in it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct ExecutionContext { - pub(crate) kind: ContextKind, - /// Whether it runs at query time (a query, or a query-time shareable - /// candidate's inputs): its `PerContext` candidates are computed in it. - pub(crate) at_query_time: bool, - /// The candidates that may give a result slot in it, ascending: each - /// computed in it ([`ExecutionContext::computes`]) or read across its - /// boundary (a shareable candidate, or a candidate of an `I_r`). - pub(crate) options: Vec, -} - -impl ExecutionContext { - /// Whether a candidate of `kind` is computed in this context (otherwise - /// it is read across the boundary). - pub(crate) fn computes(&self, kind: RunKind) -> bool { - match kind { - RunKind::PerContext => self.at_query_time, - RunKind::Ingestion => matches!(self.kind, ContextKind::Ingestion(_)), - RunKind::Shareable => false, - } - } -} - -impl MilpInputs { - /// The context where context `context` reads a candidate of an `I_r`: - /// itself for an `I_r`, `I_{r(m)}` for ingestion-time shareable `m`'s - /// context; `None` for any other context (it reads none). - pub(crate) fn ingestion_context_read_by(&self, context: usize) -> Option { - match self.contexts[context].kind { - ContextKind::Ingestion(_) => Some(context), - ContextKind::Shared(m) => self.ingestion_context_of_shareable.get(&m).copied(), - ContextKind::Query(_) => None, - } - } -} - -/// [`execution_contexts`]' result (see [`MilpInputs`] for each field). -pub(crate) struct ExecutionContexts { - pub(crate) contexts: Vec, - pub(crate) context_of_shareable: BTreeMap, - pub(crate) ingestion_context_of_shareable: BTreeMap, -} - -/// The execution contexts and their options (module documentation): from -/// each query's output candidates, each shareable candidate's producers -/// and, per ingestion pipeline `I_r`, every candidate of `I_r` an -/// ingestion-time shareable candidate of refresh window `r` -/// (`refresh_windows`) reads across its boundary, expanded through the -/// inputs of the candidates each context computes. Candidates `usable` -/// rejects (no usage) are left out. A query-time `PerContext` candidate -/// that only an ingestion-time context could read is left out too: it -/// cannot run there. -pub(crate) fn execution_contexts( - physical: &PhysicalASAPMemoCandidates, - kind: &[RunKind], - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], - refresh_windows: &HashMap>, - usable: impl Fn(PhysicalCandidateId) -> bool, -) -> ExecutionContexts { - let query_count = physical.query_roots.len(); - let output_refs = output_references(&physical.query_roots); - // Expands `seeds` in a context: the options and, through the candidates - // it computes, their producers. - let expand = |context: &ExecutionContext, seeds: Vec| { - let mut seen: BTreeSet = BTreeSet::new(); - let mut stack = seeds; - while let Some(class) = stack.pop() { - if !usable(class) || seen.contains(&class) { - continue; - } - let class_kind = kind[class.0]; - if class_kind == RunKind::PerContext && !context.at_query_time { - continue; - } - seen.insert(class); - if context.computes(class_kind) { - stack.extend( - producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied(), - ); - } - } - seen.into_iter().collect::>() - }; - let mut contexts: Vec = Vec::new(); - for query in 0..query_count { - let mut context = ExecutionContext { - kind: ContextKind::Query(query), - at_query_time: true, - options: Vec::new(), - }; - let seeds = output_refs - .iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .flat_map(|(reference, _)| output_candidates[reference].iter().copied()) - .collect(); - context.options = expand(&context, seeds); - contexts.push(context); - } - // Shareable contexts, discovered from the options, then each `I_r` from - // every candidate of `I_r` read across a boundary; repeated until - // nothing new. - let mut context_of_shareable: BTreeMap = BTreeMap::new(); - let mut shared: Vec = Vec::new(); - // Per `I_r`: its context and its seeds. - let mut ingestion: BTreeMap< - IngestionPipeline, - (ExecutionContext, BTreeSet), - > = BTreeMap::new(); - let mut scanned = 0; - loop { - let mut pending: Vec = Vec::new(); - let mut seeds: Vec<(IngestionPipeline, PhysicalCandidateId)> = Vec::new(); - let unscanned = contexts.iter().chain(shared.iter()).skip(scanned); - for context in unscanned.chain(ingestion.values().map(|(context, _)| context)) { - // Only an ingestion-time shareable candidate reads a candidate - // of an `I_r` (Stage 2's timing rule). - let pipeline = match context.kind { - ContextKind::Shared(m) if !context.at_query_time => { - Some(ingestion_pipeline(m, refresh_windows)) - } - _ => None, - }; - for &class in &context.options { - match kind[class.0] { - RunKind::Shareable => { - if !context_of_shareable.contains_key(&class) && !pending.contains(&class) { - pending.push(class); - } - } - RunKind::Ingestion => { - if let Some(pipeline) = pipeline { - seeds.push((pipeline, class)); - } - } - RunKind::PerContext => {} - } - } - } - scanned = contexts.len() + shared.len(); - let mut grown: BTreeSet = BTreeSet::new(); - for (pipeline, class) in seeds { - let (_, known) = ingestion.entry(pipeline).or_insert_with(|| { - let context = ExecutionContext { - kind: ContextKind::Ingestion(pipeline), - at_query_time: false, - options: Vec::new(), - }; - (context, BTreeSet::new()) - }); - if known.insert(class) { - grown.insert(pipeline); - } - } - if pending.is_empty() && grown.is_empty() { - break; - } - pending.sort_unstable(); - for class in pending { - let at_query_time = physical.candidates[class.0].timing == ExecutionTiming::QueryTime; - let mut context = ExecutionContext { - kind: ContextKind::Shared(class), - at_query_time, - options: Vec::new(), - }; - context.options = expand( - &context, - producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied() - .collect(), - ); - context_of_shareable.insert(class, usize::MAX); - shared.push(context); - } - for pipeline in grown { - let (context, known) = ingestion.get_mut(&pipeline).expect("seeded above"); - context.options = expand(context, known.iter().copied().collect()); - } - } - shared.sort_by_key(|context| context.kind); - let mut context_of_pipeline: BTreeMap = BTreeMap::new(); - for (pipeline, (context, _)) in ingestion { - context_of_pipeline.insert(pipeline, contexts.len()); - contexts.push(context); - } - let mut ingestion_context_of_shareable = BTreeMap::new(); - for context in shared { - if let ContextKind::Shared(class) = context.kind { - context_of_shareable.insert(class, contexts.len()); - if !context.at_query_time { - let pipeline = ingestion_pipeline(class, refresh_windows); - if let Some(&at) = context_of_pipeline.get(&pipeline) { - ingestion_context_of_shareable.insert(class, at); - } - } - } - contexts.push(context); - } - ExecutionContexts { - contexts, - context_of_shareable, - ingestion_context_of_shareable, - } -} - -// --------------------------------------------------------------------------- -// Per-candidate data -// --------------------------------------------------------------------------- - -/// Whether `node` outputs approximate values ([`CandidateData::approximate`]). -fn outputs_approximate_values(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - } | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(..), - .. - } - ) - ) -} - -/// The physical-DAG node of `candidate`, with node id `candidate` -/// and each input's child id its first producer candidate (`usize::MAX` when it -/// has none), as Stage 2's realizer would export it before retention. -fn catalog_node( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, - class: PhysicalCandidateId, -) -> PhysicalASAPDAGNode { - let candidate = &physical.candidates[class.0]; - let node = physical.physical_node(class); - let mut input = 0; - let payload = node.operator.map_children(|_| { - let child = links.producers_per_input[class.0] - .get(input) - .and_then(|producers| producers.first()) - .map_or(usize::MAX, |producer| producer.0); - input += 1; - child - }); - PhysicalASAPDAGNode { - id: class.0, - payload, - output_state: ExecutionDataState { - timing: candidate.timing, - primitive: if node.asap().and_then(ASAPOp::produced_state).is_some() { - DataPrimitive::SummaryState - } else { - DataPrimitive::Raw - }, - }, - materialization: candidate.materialization, - output_schema: node.schema.clone(), - guarantee: node.guarantee.clone(), - retention_ms: None, - } -} - -/// A DAG of `nodes` alone, without edges: what -/// [`asap_cost::tumbling_window_retention_ms`] reads (payload children and nodes). -fn nodes_dag(nodes: &[&PhysicalASAPDAGNode]) -> PhysicalASAPDAG { - PhysicalASAPDAG { - nodes: nodes.iter().map(|&node| node.clone()).collect(), - edges: Vec::new(), - roots: Vec::new(), - scalar_roots: Vec::new(), - } -} - -/// The window a scan read through `path` (time shifts, then the reader that -/// ends it) covers, as [`PhysicalASAPDAG::scan_window`] computes it for a -/// plan where `path` is the scan's only reader: the shifts' offsets add up, -/// and a `TimeRange` reads its range, a kept pane chain one pane, any other -/// pane chain its whole window; any other reader reads `Unbounded`. -fn window_read_through(path: &[&PhysicalASAPDAGNode]) -> ScanWindow { - let mut end_offset_ms: i64 = 0; - for node in path { - let width_ms: u128 = match &node.payload { - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => { - end_offset_ms = end_offset_ms.saturating_add(shift.offset_ms); - continue; - } - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => range.as_millis(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) if matches!( - node.materialization, - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) => - { - u128::from(*tumbling_window_width_ms) - } - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => u128::from(*tumbling_window_width_ms) * u128::from(*tumbling_window_count), - _ => return ScanWindow::Unbounded, - }; - return ScanWindow::Within { - end_offset_ms, - width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), - }; - } - ScanWindow::Unbounded -} - -/// The windows query-time scan `scan` may be read at: each reader's -/// windows (one per path through time shifts) and, per execution context, -/// the unions of the window kinds its readers there read (one scan run read -/// by several readers reads the union, `PhysicalASAPDAG::scan_window`): -/// every union of the kinds. -/// A read as an output or as a shareable candidate's own input is -/// `Unbounded`. Distinct; `Unbounded` alone when nothing reads it. -fn scan_window_options( - scan: PhysicalCandidateId, - windows_per_reader: &[(PhysicalCandidateId, Vec)], - contexts: &[ExecutionContext], - kind: &[RunKind], - producers_per_input: &[Vec>], - output_candidates: &[Vec], -) -> Vec { - use crate::milp::cost_terms::window_union; - let mut windows: Vec = Vec::new(); - let add = |window: ScanWindow, windows: &mut Vec| { - if !windows.contains(&window) { - windows.push(window); - } - }; - for (_, read) in windows_per_reader { - for &window in read { - add(window, &mut windows); - } - } - let window_of = |reader: PhysicalCandidateId| { - windows_per_reader - .iter() - .find(|(known, _)| *known == reader) - .map_or(ScanWindow::Unbounded, |(_, read)| { - window_union(read.iter().copied()) - }) - }; - let shareable = kind[scan.0] == RunKind::Shareable; - let mut all_kinds: Vec = Vec::new(); - for context in contexts { - if !context.options.contains(&scan) { - continue; - } - let mut kinds: Vec = Vec::new(); - for &reader in &context.options { - if context.computes(kind[reader.0]) - && producers_per_input[reader.0] - .iter() - .any(|producers| producers.contains(&scan)) - { - let window = window_of(reader); - if !kinds.contains(&window) { - kinds.push(window); - } - } - } - // A shareable candidate reading the scan as its own input reads it - // at its own window; a read as an output reads it whole. - if let ContextKind::Shared(own) = context.kind { - if producers_per_input[own.0] - .iter() - .any(|producers| producers.contains(&scan)) - { - let window = window_of(own); - if !kinds.contains(&window) { - kinds.push(window); - } - } - } - let read_whole = match context.kind { - ContextKind::Query(_) => output_candidates.iter().any(|out| out.contains(&scan)), - ContextKind::Shared(_) | ContextKind::Ingestion(_) => false, - }; - if read_whole && !kinds.contains(&ScanWindow::Unbounded) { - kinds.push(ScanWindow::Unbounded); - } - if shareable { - for window in kinds { - if !all_kinds.contains(&window) { - all_kinds.push(window); - } - } - continue; - } - add_unions(&kinds, &mut windows); - } - if shareable { - add_unions(&all_kinds, &mut windows); - } - if windows.is_empty() { - windows.push(ScanWindow::Unbounded); - } - windows -} - -/// Whether window `outer` contains window `inner`. -fn window_contains(outer: ScanWindow, inner: ScanWindow) -> bool { - match (outer, inner) { - (ScanWindow::Unbounded, _) => true, - (ScanWindow::Within { .. }, ScanWindow::Unbounded) => false, - ( - ScanWindow::Within { - end_offset_ms: outer_end, - width_ms: outer_width, - }, - ScanWindow::Within { - end_offset_ms: inner_end, - width_ms: inner_width, - }, - ) => { - outer_end <= inner_end - && outer_end.saturating_add(outer_width) >= inner_end.saturating_add(inner_width) - } - } -} - -/// Adds to `windows` every union of a nonempty subset of `kinds`. -fn add_unions(kinds: &[ScanWindow], windows: &mut Vec) { - use crate::milp::cost_terms::scan_window_read_at as window_union; - let mut push = |window: ScanWindow| { - if !windows.contains(&window) { - windows.push(window); - } - }; - // Every union of the kinds is enumerated (#1185: no cap); a shift past - // the word size would silently enumerate nothing. - assert!( - kinds.len() < usize::BITS as usize, - "a scan run with {} window kinds has too many unions to enumerate", - kinds.len() - ); - for set in 1usize..(1 << kinds.len()) { - push(window_union( - (0..kinds.len()) - .filter(|bit| set & (1 << bit) != 0) - .map(|bit| kinds[bit]), - )); - } -} - -/// Per class reading the query-time scan `scan`, the windows it may read it -/// at (through time shifts above it), distinct. -fn scan_windows_per_reader( - scan: PhysicalCandidateId, - nodes: &[PhysicalASAPDAGNode], - readers: &[Vec<(PhysicalCandidateId, usize)>], -) -> Vec<(PhysicalCandidateId, Vec)> { - /// Time shifts nested deeper than this are not followed. - const MAX_SHIFTS: usize = 4; - let mut per_reader: Vec<(PhysicalCandidateId, Vec)> = Vec::new(); - let mut paths: Vec> = vec![vec![scan]]; - while let Some(path) = paths.pop() { - let last = *path.last().expect("never empty"); - let follow = path.len() == 1 - || (matches!( - nodes[last.0].payload, - Operator::NonASAP(NonASAPOp::TimeShift { .. }) - ) && path.len() <= MAX_SHIFTS - && !readers[last.0].is_empty()); - if !follow { - // The path ends at a reader: what the scan reads under it. - let through: Vec<_> = path[1..].iter().map(|class| &nodes[class.0]).collect(); - let window = window_read_through(&through); - let Some(&reader) = path.get(1) else { - continue; - }; - match per_reader.iter_mut().find(|(known, _)| *known == reader) { - Some((_, windows)) if windows.contains(&window) => {} - Some((_, windows)) => windows.push(window), - None => per_reader.push((reader, vec![window])), - } - continue; - } - for &(reader, _) in readers[last.0].iter().rev() { - if !path.contains(&reader) { - let mut longer = path.clone(); - longer.push(reader); - paths.push(longer); - } - } - } - per_reader.sort_by_key(|(reader, _)| *reader); - per_reader -} - -/// The panes kept pane chain `chain` may keep, one per query that can reach -/// it (its range-evaluation span), in `reaching` order; its pane count for -/// any other node. -fn retained_pane_options( - chain: PhysicalCandidateId, - nodes: &[PhysicalASAPDAGNode], - first_producer: &[Option], - reaching: &[usize], - entries: &[QueryWorkloadEntry], -) -> Vec { - let node = &nodes[chain.0]; - let kept = matches!( - (&node.payload, node.materialization), - ( - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - Materialization::MatAtIngestion(_) - | Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) - ); - if !kept { - return vec![asap_cost::tumbling_windows_of(node)]; - } - // The chain over the time shifts below it, which set its input offset. - let mut below = Vec::new(); - let mut current = first_producer[chain.0]; - while let Some(producer) = current { - if !matches!( - nodes[producer.0].payload, - Operator::NonASAP(NonASAPOp::TimeShift { .. }) - ) || below.contains(&producer) - { - break; - } - below.push(producer); - current = first_producer[producer.0]; - } - let mut dag_nodes: Vec<&PhysicalASAPDAGNode> = - below.iter().rev().map(|class| &nodes[class.0]).collect(); - dag_nodes.push(node); - let dag = nodes_dag(&dag_nodes); - let mut options = Vec::new(); - let reach = vec![HashSet::from([chain.0])]; - let spans: Vec<_> = match reaching.is_empty() { - true => vec![None], - false => reaching - .iter() - .map(|&query| entries.get(query).and_then(|entry| entry.range_evaluation)) - .collect(), - }; - for range in spans { - let retention = asap_cost::tumbling_window_retention_ms(&dag, &reach, &[range]); - let panes = asap_cost::retained_tumbling_windows(node, &retention); - options.push(panes); - } - options -} - -/// Whether two producer outputs cost their readers the same. -fn same_producer_output(left: &ProducerOutput<'_>, right: &ProducerOutput<'_>) -> bool { - left.output.rows == right.output.rows - && left.output.bytes == right.output.bytes - && left.sources == right.sources - && left.family == right.family - && left.unchanged_columns == right.unchanged_columns - && (std::ptr::eq(left.payload, right.payload) - || left.payload.map_children(|_| 0usize) == right.payload.map_children(|_| 0usize)) - && (std::ptr::eq(left.output_schema, right.output_schema) - || left.output_schema == right.output_schema) -} - -/// `usage` as a reader of `node` sees it. -fn producer_output<'a>(node: &'a PhysicalASAPDAGNode, usage: &NodeUsage<'a>) -> ProducerOutput<'a> { - ProducerOutput { - payload: &node.payload, - output_schema: &node.output_schema, - output: usage.output, - sources: usage.sources.clone(), - family: usage.family, - unchanged_columns: usage.unchanged_columns.clone(), - } -} - -/// Per candidate, every `(reader candidate, input)` that may read it, ascending. -fn readers_of( - producer_candidates_per_input: &[Vec>], -) -> Vec> { - let mut readers = vec![Vec::new(); producer_candidates_per_input.len()]; - for (reader, inputs) in producer_candidates_per_input.iter().enumerate() { - for (input, producers) in inputs.iter().enumerate() { - for producer in producers { - readers[producer.0].push((PhysicalCandidateId(reader), input)); - } - } - } - readers -} - -/// Each candidate's [`CandidateData`] (module documentation, "Usage per execution -/// and its context"). `reaching[k]` are the queries that can reach candidate -/// `k` before pruning. -#[allow(clippy::too_many_arguments)] -fn candidate_data( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, - entries: &[QueryWorkloadEntry], - reaching: &[Vec], - contexts: &[ExecutionContext], - output_candidates: &[Vec], - data: &DataWorkload, - cost_inputs: &CostInputs, -) -> Vec { - let count = physical.candidates.len(); - let formula_scale = cost_inputs.frozen_calibration_factor.unwrap_or(1.0); - let nodes: Vec = (0..count) - .map(|class| catalog_node(physical, links, PhysicalCandidateId(class))) - .collect(); - let node_by_id: asap_cost::NodeMap<'_> = nodes.iter().map(|node| (node.id, node)).collect(); - let first_producer: Vec> = links - .producers_per_input - .iter() - .map(|producers_per_input| { - producers_per_input - .first() - .and_then(|producers| producers.first().copied()) - }) - .collect(); - let first_producers_of: HashMap> = links - .producers_per_input - .iter() - .enumerate() - .map(|(class, producers_per_input)| { - ( - class, - producers_per_input - .iter() - .filter_map(|producers| producers.first().map(|producer| producer.0)) - .collect(), - ) - }) - .collect(); - let readers = readers_of(&links.producers_per_input); - // Per candidate, the distinct outputs it may give its readers; `None` until - // it is costed. - let mut outputs: Vec>, String>>> = vec![None; count]; - let mut result: Vec = (0..count) - .map(|class| { - let node = physical.physical_node(PhysicalCandidateId(class)); - let operator_node = physical.candidates[class].operator_node; - let accuracy_budget_parts = physical.operator_nodes_of_candidates_for_memo_groups - [operator_node.0] - .accuracy_budget_parts; - CandidateData { - approximate: outputs_approximate_values(node), - accuracy_budget_parts, - usage: Err("not costed".to_string()), - ..CandidateData::default() - } - }) - .collect(); - // Producers first (Kahn's order). Candidate links can close a cycle - // (e.g. a widened scan's `Project` that reads its own memo group): when no candidate - // is ready, the smallest one each of whose inputs has a costed producer - // is costed from those producers alone, and marked - // [`UsageContext::ProducerOnACycle`]. - let producer_sets: Vec> = links - .producers_per_input - .iter() - .map(|producers_per_input| producers_per_input.iter().flatten().map(|p| p.0).collect()) - .collect(); - let mut waiting: Vec = producer_sets.iter().map(BTreeSet::len).collect(); - let mut ready: std::collections::BinaryHeap> = (0..count) - .filter(|&class| waiting[class] == 0) - .map(std::cmp::Reverse) - .collect(); - let mut costed_classes = vec![false; count]; - loop { - let class = match ready.pop() { - Some(std::cmp::Reverse(class)) => class, - None => match (0..count).find(|&class| { - !costed_classes[class] - && links.producers_per_input[class] - .iter() - .all(|producers| producers.iter().any(|p| costed_classes[p.0])) - }) { - Some(class) => class, - None => break, - }, - }; - if std::mem::replace(&mut costed_classes[class], true) { - continue; - } - let mut readers_of_class: BTreeSet = BTreeSet::new(); - for &(reader, _) in &readers[class] { - readers_of_class.insert(reader.0); - } - for reader in readers_of_class { - waiting[reader] -= 1; - if waiting[reader] == 0 && !costed_classes[reader] { - ready.push(std::cmp::Reverse(reader)); - } - } - let class = PhysicalCandidateId(class); - let node = &nodes[class.0]; - let inputs = &links.producers_per_input[class.0]; - // Per input, the distinct outputs of its costed producer candidates. - let mut producer_options: Vec>> = Vec::with_capacity(inputs.len()); - let mut failed = None; - let mut on_a_cycle = false; - // A scan's outputs at windows that do not contain this reader's own - // window are never what it reads (the scan reads at least the - // union of its readers' windows). - let reader = class; - let reads_output = - |result: &[CandidateData], producer: PhysicalCandidateId, at: usize| -> bool { - let windowed = &result[producer.0].usage_per_window; - if windowed.is_empty() { - return true; - } - let Some((_, own)) = result[producer.0] - .windows_per_reader - .iter() - .find(|(known, _)| *known == reader) - else { - return true; - }; - let own = crate::milp::cost_terms::window_union(own.iter().copied()); - // An unbounded read is priced at the other readers' widest - // window (`plan_cost`), whatever it is. - if own == ScanWindow::Unbounded { - return true; - } - windowed - .iter() - .any(|&(window, _, output)| output == at && window_contains(window, own)) - }; - for producers in inputs { - let mut options: Vec> = Vec::new(); - for producer in producers { - match &outputs[producer.0] { - Some(Ok(produced)) => { - for (at, output) in produced.iter().enumerate() { - if !reads_output(&result, *producer, at) { - continue; - } - if !options - .iter() - .any(|known| same_producer_output(known, output)) - { - options.push(output.clone()); - } - } - } - // An uncostable producer is one no plan uses: the - // others still serve this input. - Some(Err(_)) => {} - None => on_a_cycle = true, - } - } - if options.is_empty() { - failed.get_or_insert_with(|| { - format!( - "input {} has no producer with a usage (producers {:?})", - producer_options.len(), - producers.iter().map(|p| p.0).collect::>() - ) - }); - } - producer_options.push(options); - } - if let Some(reason) = failed { - outputs[class.0] = Some(Err(reason.clone())); - result[class.0].usage = Err(reason); - continue; - } - let windows = match (&node.payload, node.output_state.timing) { - (Operator::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => { - result[class.0].windows_per_reader = - scan_windows_per_reader(class, &nodes, &readers); - scan_window_options( - class, - &result[class.0].windows_per_reader, - contexts, - &links.kind, - &links.producers_per_input, - output_candidates, - ) - } - _ => vec![ScanWindow::Unbounded], - }; - let retained_per_query = - retained_pane_options(class, &nodes, &first_producer, &reaching[class.0], entries); - let mut retained: Vec = Vec::new(); - for &panes in &retained_per_query { - if !retained.contains(&panes) { - retained.push(panes); - } - } - if retained.len() > 1 { - result[class.0].retained_panes_per_query = reaching[class.0] - .iter() - .copied() - .zip(retained_per_query.iter().copied()) - .collect(); - } - let ingestion_window_ms = (node.output_state.timing == ExecutionTiming::IngestionTime) - .then(|| asap_cost::maintained_window_ms(class.0, &node_by_id, &first_producers_of)) - .flatten(); - // Every combination of the options. - let combinations = producer_options - .iter() - .map(Vec::len) - .chain([windows.len(), retained.len()]) - .try_fold(1usize, |total, options| total.checked_mul(options)) - .expect("the number of context combinations fits in usize"); - // Each costed combination: its usage, its output, and its picks - // (the option per input, the window and the retained panes). - let mut costed: Vec<(CandidateUsage, ProducerOutput<'_>, Picks)> = Vec::new(); - let mut error = None; - for index in 0..combinations { - let mut rest = index; - let mut pick = |options: usize| { - let picked = rest % options; - rest /= options; - picked - }; - let retained_pick = pick(retained.len()); - let retained_tumbling_windows = retained[retained_pick]; - let window_pick = pick(windows.len()); - let window = windows[window_pick]; - let mut option_picks: Vec = producer_options - .iter() - .rev() - .map(|options| pick(options.len())) - .collect(); - option_picks.reverse(); - let producers: Vec> = producer_options - .iter() - .zip(&option_picks) - .map(|(options, &picked)| options[picked].clone()) - .collect(); - let usage_at = |per_second: f64| { - asap_cost::node_usage( - node, - &producers, - (1, per_second), - window, - retained_tumbling_windows, - ingestion_window_ms, - data, - cost_inputs, - formula_scale, - ) - }; - match (usage_at(1.0), usage_at(0.0)) { - (Ok(executed), Ok(standing)) => { - let usage = candidate_usage(&executed, &standing, node.output_state.timing); - costed.push(( - usage, - producer_output(node, &executed), - Picks { - options: option_picks, - window: window_pick, - retained: retained_pick, - }, - )); - } - (Err(e), _) | (_, Err(e)) => { - error.get_or_insert(e.to_string()); - } - } - } - // A context `asap-cost` cannot cost is one no plan can use. - let uncostable_context = error.is_some(); - if costed.is_empty() { - let error = error.unwrap_or_else(|| "no context".into()); - outputs[class.0] = Some(Err(error.clone())); - result[class.0].usage = Err(error); - continue; - } - let first = costed[0].0; - let mut depends_on = Vec::new(); - // Its distinct outputs, and the one each costed combination gives. - let mut produced: Vec> = Vec::new(); - let mut output_of_combination = Vec::with_capacity(costed.len()); - for (_, output, _) in &costed { - let at = match produced - .iter() - .position(|known| same_producer_output(known, output)) - { - Some(at) => at, - None => { - produced.push(output.clone()); - produced.len() - 1 - } - }; - output_of_combination.push(at); - } - // What a combination gives its plan: its usage and its output. - let given: Vec<(CandidateUsage, usize)> = costed - .iter() - .zip(&output_of_combination) - .map(|((usage, _, _), &output)| (*usage, output)) - .collect(); - // The inputs whose option changes the usage or the output, and - // whether the window does: two costed combinations differing only - // there give different usages or outputs. - let changes = |differs: &dyn Fn(&Picks, &Picks) -> bool| { - costed.iter().zip(&given).any(|((_, _, picks), gives)| { - costed - .iter() - .zip(&given) - .any(|((_, _, others), other)| differs(picks, others) && gives != other) - }) - }; - let inputs_changing_usage: Vec = (0..producer_options.len()) - .filter(|&input| { - changes(&|a: &Picks, b: &Picks| { - a.window == b.window - && a.retained == b.retained - && a.options[input] != b.options[input] - && (0..a.options.len()) - .all(|other| other == input || a.options[other] == b.options[other]) - }) - }) - .collect(); - let window_changes_usage = changes(&|a: &Picks, b: &Picks| { - a.window != b.window && a.retained == b.retained && a.options == b.options - }); - if !inputs_changing_usage.is_empty() { - // Per changing input, each producer and the options it gives: - // one choice per output of a producer with several. - let options_of_producers: Vec)>> = - inputs_changing_usage - .iter() - .map(|&input| { - let mut choices = Vec::new(); - for &producer in &inputs[input] { - let Some(Ok(produced)) = &outputs[producer.0] else { - continue; - }; - let option_of = |output: &ProducerOutput<'_>| { - producer_options[input] - .iter() - .position(|known| same_producer_output(known, output)) - }; - if produced.len() == 1 { - choices.push(( - ProducerChoice { - class: producer, - output: None, - }, - produced.iter().filter_map(option_of).collect(), - )); - continue; - } - for (at, output) in produced.iter().enumerate() { - if !reads_output(&result, producer, at) { - continue; - } - choices.push(( - ProducerChoice { - class: producer, - output: Some(at), - }, - option_of(output).into_iter().collect(), - )); - } - } - choices - }) - .collect(); - let choices = options_of_producers - .iter() - .try_fold(1usize, |total, producers| { - total.checked_mul(producers.len()) - }) - .expect("the number of producer choices fits in usize"); - for index in 0..choices { - let mut rest = index; - let chosen: Vec<&(ProducerChoice, Vec)> = options_of_producers - .iter() - .map(|producers| { - let picked = &producers[rest % producers.len()]; - rest /= producers.len(); - picked - }) - .collect(); - let matching: Vec<&(CandidateUsage, usize)> = costed - .iter() - .zip(&given) - .filter(|((_, _, picks), _)| { - inputs_changing_usage - .iter() - .zip(&chosen) - .all(|(&input, (_, options))| options.contains(&picks.options[input])) - }) - .map(|(_, gives)| gives) - .collect(); - let costliest = matching.iter().copied().max_by(|a, b| { - let total = |usage: &CandidateUsage| { - usage.cost_per_execution.unwrap_or(0.0) - + usage.standing_cost_per_second.unwrap_or(0.0) - + usage.work_ms_per_execution - }; - total(&a.0).total_cmp(&total(&b.0)) - }); - if matching.iter().any(|gives| Some(*gives) != costliest) { - result[class.0].producer_usage_ambiguous = true; - } - result[class.0].usage_per_producers.push(( - chosen.iter().map(|(choice, _)| *choice).collect(), - costliest.copied(), - )); - } - result[class.0].inputs_changing_usage = inputs_changing_usage; - } - if retained.len() > 1 { - let retained_changes_usage = changes(&|a: &Picks, b: &Picks| { - a.retained != b.retained && a.window == b.window && a.options == b.options - }); - if retained_changes_usage { - for (pick, &panes) in retained.iter().enumerate() { - if let Some(at) = costed - .iter() - .position(|(_, _, picks)| picks.retained == pick) - { - result[class.0] - .usage_per_retained_panes - .push((panes, given[at].0)); - } - } - } else { - result[class.0].retained_panes_per_query.clear(); - } - } - // A query-time scan keeps its usage per window even when every - // costed window gives the same: a window `asap-cost` cannot cost - // (an unbounded read with no `rows_held`) must stay unusable. - let query_time_scan = matches!( - (&node.payload, node.output_state.timing), - ( - Operator::NonASAP(NonASAPOp::Scan { .. }), - ExecutionTiming::QueryTime - ) - ); - if window_changes_usage || query_time_scan { - for (pick, window) in windows.iter().enumerate() { - if let Some(at) = costed.iter().position(|(_, _, picks)| picks.window == pick) { - let (usage, output) = given[at]; - result[class.0] - .usage_per_window - .push((*window, usage, output)); - } - } - } - if given.iter().any(|gives| gives.0 != first) { - if producer_options.iter().any(|options| options.len() > 1) { - depends_on.push(UsageContext::ProducerOutput); - } - if windows.len() > 1 { - depends_on.push(UsageContext::ScanWindowOfReader); - } - if retained.len() > 1 { - depends_on.push(UsageContext::PaneRetentionOfReachingQueries); - } - } - if on_a_cycle { - depends_on.push(UsageContext::ProducerOnACycle); - } - if uncostable_context { - depends_on.push(UsageContext::UncostableContext); - } - result[class.0].output_count = produced.len(); - outputs[class.0] = Some(Ok(produced)); - result[class.0].usage = Ok(first); - result[class.0].usage_depends_on = depends_on; - } - for (class, costed) in costed_classes.iter().enumerate() { - if !costed { - result[class].usage = Err("its producer links form a cycle".into()); - } - } - result -} - -/// One costed combination of a class's context options, by index: the -/// option per input, the window and the retained panes. -#[derive(Debug, Clone, PartialEq, Eq)] -struct Picks { - options: Vec, - window: usize, - retained: usize, -} - -/// The numbers of one execution (`executed`, at one execution per second) -/// and of none (`standing`) of a node at `timing`. -fn candidate_usage( - executed: &NodeUsage<'_>, - standing: &NodeUsage<'_>, - timing: ExecutionTiming, -) -> CandidateUsage { - let standing_cost = standing.cost.cost_per_second; - CandidateUsage { - cpu_ops_per_execution: executed.cpu_ops_per_second, - scanned_bytes_per_execution: executed.scanned_bytes_per_second, - work_ms_per_execution: executed.work_ms_per_execution, - own_latency_ms: match timing { - ExecutionTiming::QueryTime => executed.work_ms_per_execution, - ExecutionTiming::IngestionTime => 0.0, - }, - cost_per_execution: executed - .cost - .cost_per_second - .zip(standing_cost) - .map(|(total, standing)| total - standing), - standing_cost_per_second: standing_cost, - average_memory_bytes: executed.average_memory_bytes, - peak_memory_bytes: executed.peak_memory_bytes, - disk_bytes: executed.disk_bytes, - held_bytes: executed.held.map_or(0, |(bytes, _, _)| bytes), - raw_data_kept_bytes: executed.raw_scan.map_or(0, |(_, bytes)| bytes), - maintained_window_bytes: executed.maintained.unwrap_or(0), - output_rows: executed.output.rows, - output_bytes: executed.output.bytes, - } -} - -// --------------------------------------------------------------------------- -// Dominance -// --------------------------------------------------------------------------- - -/// What one candidate is compared on (module documentation, "Dominance"). -#[derive(Debug, Clone)] -pub(crate) struct DominanceProfile<'a> { - /// Must be equal for one candidate to dominate another. - pub(crate) same: DominanceKey<'a>, - pub(crate) guarantee: Option<&'a AccuracyGuarantee>, -} - -/// The parts of a [`DominanceProfile`] that must be equal. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct DominanceKey<'a> { - pub(crate) group: MemoGroupId, - pub(crate) timing: ExecutionTiming, - pub(crate) materialization: Materialization, - pub(crate) shareable: bool, - pub(crate) producer_candidates_per_input: &'a [Vec], - pub(crate) readers: Vec<(PhysicalCandidateId, usize)>, - pub(crate) output_references: Vec, - pub(crate) reader_constraint_ids: Vec, - pub(crate) may_keep_across_evaluations_per_sole_reader_query: &'a [bool], - pub(crate) output_schema: &'a Schema, - pub(crate) operator_kind: OperatorKindForReaders, -} - -/// The parts of a node's operator its readers' costs read: its kind, and a -/// pane chain's panes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum OperatorKindForReaders { - SummaryTumblingWindowChain { - tumbling_window_width_ms: u64, - tumbling_window_count: u32, - }, - Other(std::mem::Discriminant, &'static str), -} - -/// Each candidate's [`DominanceProfile`]. `output_candidates` are `Out(r)` before -/// pruning. -fn dominance_profiles<'a>( - physical: &'a PhysicalASAPMemoCandidates, - links: &'a CandidateLinks, - output_candidates: &[Vec], -) -> Vec> { - let constraint_ids = reader_constraint_ids_per_candidate(physical); - let mut readers = readers_of(&links.producers_per_input); - for list in &mut readers { - list.sort_unstable(); - } - let mut output_refs_of = vec![Vec::new(); physical.candidates.len()]; - for (reference, out) in output_candidates.iter().enumerate() { - for class in out { - output_refs_of[class.0].push(reference); - } - } - links - .producers_per_input - .iter() - .enumerate() - .map(|(class, producers_per_input)| { - let copy = PhysicalCandidateId(class); - let candidate = &physical.candidates[copy.0]; - let node = physical.physical_node(copy); - DominanceProfile { - same: DominanceKey { - group: physical.memo_group_of(copy), - timing: candidate.timing, - materialization: candidate.materialization, - shareable: links.is_shareable(copy), - producer_candidates_per_input: producers_per_input, - readers: std::mem::take(&mut readers[class]), - output_references: std::mem::take(&mut output_refs_of[class]), - reader_constraint_ids: constraint_ids[copy.0].clone(), - may_keep_across_evaluations_per_sole_reader_query: &candidate - .may_keep_across_evaluations_per_sole_reader_query, - output_schema: &node.schema, - operator_kind: match &node.operator { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => OperatorKindForReaders::SummaryTumblingWindowChain { - tumbling_window_width_ms: *tumbling_window_width_ms, - tumbling_window_count: *tumbling_window_count, - }, - operator => OperatorKindForReaders::Other( - std::mem::discriminant(operator), - node.operator.kind_name(), - ), - }, - }, - guarantee: node.guarantee.as_ref(), - } - }) - .collect() -} - -/// Whether `stronger` is at least as strong as `weaker`: equal, or `stronger` -/// exact, or both in one metric and scope with constant bounds and failure -/// probabilities no larger. Anything else (an unknown statistic, different -/// metrics) is not comparable. -pub(crate) fn guarantee_at_least_as_strong( - stronger: Option<&AccuracyGuarantee>, - weaker: Option<&AccuracyGuarantee>, -) -> bool { - match (stronger, weaker) { - (None, None) => true, - (Some(stronger), Some(weaker)) => { - if stronger.spec == weaker.spec || stronger.is_exact() { - return true; - } - stronger.spec.accuracy_metric == weaker.spec.accuracy_metric - && stronger.spec.accuracy_scope == weaker.spec.accuracy_scope - && matches!( - (stronger.spec.error.evaluate(), weaker.spec.error.evaluate()), - (Some(s), Some(w)) if s <= w - ) - && matches!( - ( - stronger.spec.failure_probability.evaluate(), - weaker.spec.failure_probability.evaluate() - ), - (Some(s), Some(w)) if s <= w - ) - } - _ => false, - } -} - -/// Every number of `usage` that a candidate it dominates must not beat, in a -/// fixed order. -fn compared_numbers(usage: &CandidateUsage) -> [f64; 12] { - [ - usage.cpu_ops_per_execution, - usage.scanned_bytes_per_execution, - usage.work_ms_per_execution, - usage.own_latency_ms, - usage.average_memory_bytes, - usage.peak_memory_bytes as f64, - usage.disk_bytes as f64, - usage.held_bytes as f64, - usage.raw_data_kept_bytes as f64, - usage.maintained_window_bytes as f64, - usage.output_rows as f64, - usage.output_bytes as f64, - ] -} - -/// Whether candidate `by` dominates candidate `candidate` (module documentation). -pub(crate) fn dominates( - by: (&DominanceProfile<'_>, &CandidateData), - class: (&DominanceProfile<'_>, &CandidateData), -) -> bool { - let ((by_profile, by_data), (profile, data)) = (by, class); - let (Ok(by_usage), Ok(usage)) = (&by_data.usage, &data.usage) else { - return false; - }; - by_profile.same == profile.same - && by_data.usage_depends_on.is_empty() - && data.usage_depends_on.is_empty() - && by_data.approximate == data.approximate - && by_data.accuracy_budget_parts == data.accuracy_budget_parts - && guarantee_at_least_as_strong(by_profile.guarantee, profile.guarantee) - && compared_numbers(by_usage) - .iter() - .zip(compared_numbers(usage)) - .all(|(by, class)| *by <= class) -} - -/// The candidates another candidate dominates (module documentation), each with -/// the smallest dominating candidate that is kept. Among candidates that dominate -/// each other, the smallest id is kept. -pub(crate) fn prune_dominated_candidates( - profiles: &[DominanceProfile<'_>], - candidate_data: &[CandidateData], -) -> Vec { - // Only candidates with equal keys can dominate each other. - let mut buckets: Vec> = Vec::new(); - let mut bucket_of_key: HashMap<(MemoGroupId, ExecutionTiming, Materialization), Vec> = - HashMap::new(); - for (class, profile) in profiles.iter().enumerate() { - let candidates = bucket_of_key - .entry(( - profile.same.group, - profile.same.timing, - profile.same.materialization, - )) - .or_default(); - match candidates - .iter() - .find(|&&bucket| profiles[buckets[bucket][0]].same == profile.same) - { - Some(&bucket) => buckets[bucket].push(class), - None => { - candidates.push(buckets.len()); - buckets.push(vec![class]); - } - } - } - let mut pruned = Vec::new(); - for bucket in buckets.iter().filter(|bucket| bucket.len() > 1) { - for &class in bucket { - let dominated_by = bucket.iter().copied().find(|&other| { - other != class - && dominates( - (&profiles[other], &candidate_data[other]), - (&profiles[class], &candidate_data[class]), - ) - && (other < class - || !dominates( - (&profiles[class], &candidate_data[class]), - (&profiles[other], &candidate_data[other]), - )) - }); - if let Some(other) = dominated_by { - pruned.push(DominatedCandidate { - class: PhysicalCandidateId(class), - dominated_by: PhysicalCandidateId(other), - reason: format!( - "candidate {other} has the same memo group, producers, readers and \ - (approx, parts), a guarantee at least as strong, and no larger \ - usage, latency, output or held state" - ), - }); - } - } - } - pruned.sort_by_key(|pruned| pruned.class); - pruned -} - -mod accuracy_positions; - -use accuracy_positions::AccuracyPositions; - -#[cfg(test)] -mod tests; diff --git a/crates/plan-selection/src/groups/accuracy_positions.rs b/crates/plan-selection/src/groups/accuracy_positions.rs deleted file mode 100644 index fa81af61f..000000000 --- a/crates/plan-selection/src/groups/accuracy_positions.rs +++ /dev/null @@ -1,777 +0,0 @@ -//! Accuracy weights and pair exclusions per query (module documentation of -//! [`super`], "Accuracy weights"), each from `asap-accuracy`'s composition -//! and Stage 3's own check ([`guarantee_check`]) of an output built over -//! the candidates. -//! -//! For query `q` and a set `S` of approximate candidates (one candidate, or -//! a pair), the outputs `q` can have with exactly `S` approximate are built -//! from the candidates' operator nodes: a candidate in `S` is its own node; -//! a candidate with no member of `S` below it stands in as its node with an -//! exact guarantee; any other candidate is its node rebuilt over each -//! combination of its producers' outputs, with its guarantee composed again -//! ([`composed_guarantee`]), as the realizer recomposes a selected plan. -//! Outputs over which readers compose the same guarantee are kept once -//! ([`reader_key`]). Each output is then checked as -//! the selected plan is: [`guarantee_check`] with the deployment's model. -//! `S` is usable for `q` only when every output passes: a candidate that -//! fails through one reader is unusable through every reader (the largest -//! weight counts, as before). The candidates of one operator node (other -//! timings or materializations) build the same node, so they are checked -//! once, together. What a check builds is kept for the later checks -//! ([`BuiltBefore`]). -use std::cell::OnceCell; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::hash::{Hash, Hasher}; -use std::rc::Rc; - -use asap_accuracy::propagation::{composed_guarantee, root_guarantee, with_guarantees}; -use asap_types::ir::properties::AccuracyGuarantee; -use asap_types::ir::{OperatorNode, OperatorResultKind, QueryRoot}; -use asap_types::stages::{output_references, PhysicalASAPMemoCandidates, PhysicalCandidateId}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use super::{AccuracyWeight, CandidateData, UnusableReason}; -use crate::checks::guarantee_check; -use crate::report::RejectionCause; -use crate::SelectionModels; - -/// What the accuracy rows (#1095 §2.0 row 9) are built from. -#[derive(Debug, Clone, Default)] -pub(crate) struct AccuracyPositions { - /// Per query: `w(q, k)` for each approximate candidate `k` whose values - /// reach `q`'s output. - pub(crate) weights: Vec>, - /// Per query: the pairs `(k1, k2)`, `k1 < k2`, each usable alone, whose - /// outputs miss `q`'s target when both are approximate: - /// `a[q, k1] + a[q, k2] ≤ 1`. - pub(crate) excluded_pairs: Vec>, -} - -/// The outcome of one check of a query with a set of approximate candidates. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Outcome { - /// Every output is exact: no value of the set reaches the output. - Exact, - /// Every output meets the target. - Meets, - Fails(UnusableReason), -} - -/// [`AccuracyPositions`] of `physical`'s queries over the candidate links -/// left (`producer_candidates_per_input`, `output_candidates`). -pub(crate) fn accuracy_positions( - physical: &PhysicalASAPMemoCandidates, - candidate_data: &[CandidateData], - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> AccuracyPositions { - let below = approximate_below(candidate_data, producer_candidates_per_input); - let output_refs = output_references(&physical.query_roots); - let exact_stand_ins: Vec>> = (0..physical.candidates.len()) - .map(|_| OnceCell::new()) - .collect(); - let mut built_before = BuiltBefore::default(); - let mut positions = AccuracyPositions::default(); - for (query, root) in physical.query_roots.iter().enumerate() { - let references: Vec = (0..output_refs.len()) - .filter(|&reference| output_refs[reference].0 == query) - .collect(); - let reachable: BTreeSet = references - .iter() - .flat_map(|&reference| &output_candidates[reference]) - .flat_map(|class| below[class.0].iter().copied()) - .collect(); - let Some(entry) = entries.get(query) else { - positions.weights.push(BTreeMap::new()); - positions.excluded_pairs.push(Vec::new()); - continue; - }; - let checker = Checker { - physical, - exact_stand_ins: &exact_stand_ins, - producer_candidates_per_input, - output_candidates, - below: &below, - query, - root, - references: &references, - entry, - data, - models, - }; - // Candidates of one operator node (other timings or - // materializations) build the same node: checked once, together. - let mut groups: BTreeMap> = BTreeMap::new(); - for &candidate in &reachable { - groups - .entry(physical.candidates[candidate].operator_node.0) - .or_default() - .push(candidate); - } - let mut weights = BTreeMap::new(); - let mut usable: Vec<&[usize]> = Vec::new(); - for group in groups.values() { - let weight = match checker.check(group, &mut built_before) { - Outcome::Exact => continue, - Outcome::Meets => { - usable.push(group); - AccuracyWeight::One - } - Outcome::Fails(reason) => AccuracyWeight::Unusable(reason), - }; - for &candidate in group { - weights.insert(PhysicalCandidateId(candidate), weight); - } - } - let usable_candidates: Vec = usable - .iter() - .flat_map(|group| group.iter().copied()) - .collect(); - let group_of: HashMap = usable - .iter() - .enumerate() - .flat_map(|(index, group)| group.iter().map(move |&candidate| (candidate, index))) - .collect(); - let group_pairs: BTreeSet<(usize, usize)> = pairs_that_meet( - &usable_candidates, - &references, - physical, - producer_candidates_per_input, - output_candidates, - &below, - ) - .into_iter() - .map(|(first, second)| (group_of[&first], group_of[&second])) - .filter(|(first, second)| first != second) - .map(|(first, second)| (first.min(second), first.max(second))) - // A pair with a member sized for fewer than two shares is never - // read together: the shares row already excludes it. - .filter(|&(first, second)| { - [first, second] - .iter() - .all(|&group| candidate_data[usable[group][0]].accuracy_budget_parts >= 2) - }) - .collect(); - let mut excluded = Vec::new(); - for (first, second) in group_pairs { - let both: Vec = usable[first] - .iter() - .chain(usable[second]) - .copied() - .collect(); - if let Outcome::Fails(_) = checker.check(&both, &mut built_before) { - for &one in usable[first] { - for &other in usable[second] { - excluded.push(( - PhysicalCandidateId(one.min(other)), - PhysicalCandidateId(one.max(other)), - )); - } - } - } - } - excluded.sort_unstable(); - positions.weights.push(weights); - positions.excluded_pairs.push(excluded); - } - positions -} - -/// Per candidate, the approximate candidates at or below it through the -/// links, ascending. A fixpoint, so cycles of candidate links are covered. -fn approximate_below( - candidate_data: &[CandidateData], - producer_candidates_per_input: &[Vec>], -) -> Vec> { - let mut below: Vec> = candidate_data - .iter() - .enumerate() - .map(|(class, data)| match data.approximate { - true => BTreeSet::from([class]), - false => BTreeSet::new(), - }) - .collect(); - loop { - let mut changed = false; - for class in 0..below.len() { - let mut grown = below[class].clone(); - for producer in producer_candidates_per_input[class].iter().flatten() { - grown.extend(below[producer.0].iter().copied()); - } - if grown.len() != below[class].len() { - below[class] = grown; - changed = true; - } - } - if !changed { - return below; - } - } -} - -/// The pairs of `usable` (ascending, `first < second`) one plan of the -/// query can read together where their values meet: under two different -/// inputs of one candidate, under two output references of a scalar root, -/// or one below the other. -fn pairs_that_meet( - usable: &[usize], - references: &[usize], - physical: &PhysicalASAPMemoCandidates, - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], - below: &[BTreeSet], -) -> BTreeSet<(usize, usize)> { - let usable: BTreeSet = usable.iter().copied().collect(); - let mut pairs = BTreeSet::new(); - let mut add_across = |sides: &[BTreeSet]| { - for (index, left) in sides.iter().enumerate() { - for right in &sides[index + 1..] { - for &a in left { - for &b in right { - if a != b { - pairs.insert((a.min(b), a.max(b))); - } - } - } - } - } - }; - let side = |producers: &[PhysicalCandidateId]| -> BTreeSet { - producers - .iter() - .flat_map(|producer| below[producer.0].iter().copied()) - .filter(|candidate| usable.contains(candidate)) - .collect() - }; - for producers_per_input in producer_candidates_per_input - .iter() - .take(physical.candidates.len()) - { - if producers_per_input.len() < 2 { - continue; - } - let sides: Vec> = producers_per_input - .iter() - .map(|producers| side(producers)) - .collect(); - add_across(&sides); - } - let sides: Vec> = references - .iter() - .map(|&reference| side(&output_candidates[reference])) - .collect(); - add_across(&sides); - for &upper in &usable { - for &lower in &below[upper] { - if lower != upper && usable.contains(&lower) { - pairs.insert((lower.min(upper), lower.max(upper))); - } - } - } - pairs -} - -/// One query's checks. -struct Checker<'a> { - physical: &'a PhysicalASAPMemoCandidates, - /// Per candidate, its node with an exact guarantee, built once. - exact_stand_ins: &'a [OnceCell>], - producer_candidates_per_input: &'a [Vec>], - output_candidates: &'a [Vec], - below: &'a [BTreeSet], - query: usize, - root: &'a QueryRoot, - references: &'a [usize], - entry: &'a QueryWorkloadEntry, - data: &'a DataWorkload, - models: &'a SelectionModels<'a>, -} - -/// The outputs one candidate is built as, or why they are not checked. -type Built = Result>, UnusableReason>; - -/// What the checks built, kept across checks and queries. A candidate's -/// outputs depend only on the members of the approximate set at or below -/// it, so a pair check rebuilds only the candidates above both members -/// and reuses the rest from the single checks. Outputs built across a -/// cycle of links depend on where the walk entered the cycle and are not -/// kept. -#[derive(Default)] -struct BuiltBefore { - /// Per (candidate, approximate members at or below it, ascending): its - /// outputs. - outputs: HashMap<(usize, Vec), Built>, - /// The [`reader_key`]s of every node built so far. - keys: Keys, -} - -#[cfg(test)] -thread_local! { - /// Set by a test to check with no kept builds: every check builds - /// from scratch ([`BuiltBefore`] cleared) and builds every output - /// candidate (no exact stand-in skipped). - pub(crate) static FRESH_BUILDS: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - -/// One check's walk. -#[derive(Default)] -struct Walk { - /// Per candidate built in this check. - built: HashMap, - /// The candidates being built. - visiting: Vec, - /// How many reads of a candidate still being built the walk has made. - cycle_reads: usize, -} - -/// A candidate's outputs in one check, and whether they were built across -/// a cycle of links (then they are not kept in [`BuiltBefore`]). -struct BuiltInWalk { - outputs: Built, - across_cycle: bool, -} - -impl Checker<'_> { - /// Checks the query's outputs with exactly `approximate` approximate. - /// `before` holds what earlier checks built, for every query. - fn check(&self, approximate: &[usize], before: &mut BuiltBefore) -> Outcome { - #[cfg(test)] - let fresh = FRESH_BUILDS.with(std::cell::Cell::get); - #[cfg(not(test))] - let fresh = false; - if fresh { - *before = BuiltBefore::default(); - } - let mut walk = Walk::default(); - let mut per_reference: Vec>> = Vec::new(); - for &reference in self.references { - let mut outputs: Vec> = Vec::new(); - // Exact stand-ins that are not summary states are the same - // for readers ([`reader_key`]): only the first is kept. - let mut exact_value_stand_in = false; - for &class in &self.output_candidates[reference] { - let reaches = approximate - .iter() - .any(|member| self.below[class.0].contains(member)); - if !fresh - && !reaches - && self.physical.physical_node(class).result_kind != OperatorResultKind::State - && std::mem::replace(&mut exact_value_stand_in, true) - { - continue; - } - match self.built(class.0, approximate, &mut walk, before) { - Ok(built) => outputs.extend(built), - Err(reason) => return Outcome::Fails(reason), - } - } - per_reference.push(distinct_for_readers(outputs, &mut before.keys)); - } - let roots = self.roots(&per_reference); - let mut outcome = Outcome::Exact; - for root in roots { - if root_guarantee(&root).is_some_and(|g| g.is_exact()) { - continue; - } - match guarantee_check( - self.query + 1, - &root, - &self.entry.requirements.accuracy, - self.data, - self.models, - ) { - Ok(_) => outcome = Outcome::Meets, - Err(failure) => { - return Outcome::Fails(match failure.cause { - RejectionCause::NoAccuracyGuarantee => UnusableReason::NoAccuracyGuarantee, - _ => UnusableReason::AccuracyTargetMissed, - }) - } - } - } - outcome - } - - /// The query roots over each combination of `per_reference`'s outputs. - fn roots(&self, per_reference: &[Vec>]) -> Vec { - combinations(per_reference) - .into_iter() - .map(|chosen| match self.root { - QueryRoot::Operator(_) => QueryRoot::Operator(chosen[0].clone()), - QueryRoot::Scalar(expr) => { - let mut next = chosen.into_iter(); - QueryRoot::Scalar( - expr.map_operator_refs(&mut |_| { - next.next().expect("one output per reference") - }), - ) - } - }) - .collect() - } - - /// `class`'s node with an exact guarantee, built once. - fn exact_stand_in(&self, class: usize) -> Rc { - self.exact_stand_ins[class] - .get_or_init(|| { - let node = self.physical.physical_node(PhysicalCandidateId(class)); - Rc::new( - (**node) - .clone() - .with_guarantee(Some(AccuracyGuarantee::exact("an exact stand-in"))), - ) - }) - .clone() - } - - /// The outputs `class` is built as with `approximate` approximate - /// (module documentation). `walk.visiting` holds the candidates being - /// built, so a cycle of links builds nothing. - fn built( - &self, - class: usize, - approximate: &[usize], - walk: &mut Walk, - before: &mut BuiltBefore, - ) -> Built { - if let Some(done) = walk.built.get(&class) { - walk.cycle_reads += usize::from(done.across_cycle); - return done.outputs.clone(); - } - if walk.visiting.contains(&class) { - walk.cycle_reads += 1; - return Ok(Vec::new()); - } - let reaches = |candidate: usize| { - approximate - .iter() - .any(|member| self.below[candidate].contains(member)) - }; - if !reaches(class) { - return Ok(vec![self.exact_stand_in(class)]); - } - let mut members: Vec = approximate - .iter() - .copied() - .filter(|member| self.below[class].contains(member)) - .collect(); - members.sort_unstable(); - let key = (class, members); - if let Some(done) = before.outputs.get(&key) { - walk.built.insert( - class, - BuiltInWalk { - outputs: done.clone(), - across_cycle: false, - }, - ); - return done.clone(); - } - let cycle_reads = walk.cycle_reads; - let node = self.physical.physical_node(PhysicalCandidateId(class)); - let producers = &self.producer_candidates_per_input[class]; - let below_inputs = producers - .iter() - .flatten() - .any(|producer| reaches(producer.0)); - let built = if !below_inputs { - // `class` is in `approximate`, over exact inputs: its node with - // every guarantee below it composed, as the realizer does. - with_guarantees(node, &mut HashMap::new()) - .map(|node| vec![node]) - .map_err(|_| UnusableReason::NotChecked) - } else { - walk.visiting.push(class); - let mut per_input = Vec::with_capacity(producers.len()); - let mut failed = None; - for input in producers { - let mut outputs = Vec::new(); - for producer in input.iter().filter(|producer| reaches(producer.0)) { - match self.built(producer.0, approximate, walk, before) { - Ok(built) => outputs.extend(built), - Err(reason) => failed = Some(reason), - } - } - // One exact stand-in for the producers with no member below: - // one with no approximate candidate below, when there is one. - let exact = input - .iter() - .filter(|producer| !reaches(producer.0)) - .min_by_key(|producer| !self.below[producer.0].is_empty()); - if let Some(exact) = exact { - outputs.push(self.exact_stand_in(exact.0)); - } - per_input.push(distinct_for_readers(outputs, &mut before.keys)); - } - walk.visiting.pop(); - match failed { - Some(reason) => Err(reason), - None => rebuilt(node, &per_input, &mut before.keys), - } - }; - let across_cycle = walk.cycle_reads != cycle_reads; - if !across_cycle { - before.outputs.insert(key, built.clone()); - } - walk.built.insert( - class, - BuiltInWalk { - outputs: built.clone(), - across_cycle, - }, - ); - built - } -} - -/// `node` rebuilt over each combination of `per_input`'s outputs, with its -/// timing and materialization and its guarantee composed again; outputs the -/// same for their readers kept once. -fn rebuilt(node: &Rc, per_input: &[Vec>], keys: &mut Keys) -> Built { - let combinations = combinations(per_input); - let mut outputs = Vec::with_capacity(combinations.len()); - for chosen in combinations { - let mut next = chosen.into_iter(); - let rebuilt = node - .with_new_children(|_| next.next().expect("one output per input")) - .map_err(|_| UnusableReason::NotChecked)?; - let guarantee = composed_guarantee(&rebuilt); - outputs.push(Rc::new( - rebuilt - .with_guarantee(guarantee) - .with_timing(node.timing) - .with_materialization(node.materialization), - )); - } - Ok(distinct_for_readers(outputs, keys)) -} - -/// Every choice of one entry per list, in order. One empty list gives none. -fn combinations(lists: &[Vec]) -> Vec> { - let mut combinations: Vec> = vec![Vec::new()]; - for list in lists { - combinations = combinations - .into_iter() - .flat_map(|prefix| { - list.iter().map(move |item| { - let mut next = prefix.clone(); - next.push(item.clone()); - next - }) - }) - .collect(); - } - combinations -} - -/// `outputs` with outputs the same for their readers kept once (the -/// first): equal [`reader_key`]s. -fn distinct_for_readers(outputs: Vec>, keys: &mut Keys) -> Vec> { - let mut seen = HashSet::new(); - outputs - .into_iter() - .filter(|output| seen.insert(reader_key(output, keys))) - .collect() -} - -/// Per node (by address): its [`reader_key`], and whether binding -/// statistics leaves its guarantee as it is; the node is held so its -/// address is not reused during the check. -type Keys = HashMap<*const OperatorNode, (u64, bool, Rc)>; - -/// A key equal for outputs over which readers compose the same guarantee: -/// an exact node as one key (an exact summary state, a stand-in, by its -/// address); a node with a guarantee (not a summary state) by its bound and failure -/// probability (binding keeps a node's guarantee and ignores its inputs; -/// provenance is not read); a node with none that binding cannot give one -/// as one key; any other node (a summary state, whose readers read its -/// inputs' guarantees, or a node with none that binding may still give -/// one) by its operator kind and its inputs' keys. -fn reader_key(node: &Rc, keys: &mut Keys) -> u64 { - key_and_settled(node, keys).0 -} - -/// [`reader_key`], and whether binding statistics leaves `node`'s -/// guarantee as it is: a guarantee with no unknown statistic, or none over -/// inputs that are all settled. -fn key_and_settled(node: &Rc, keys: &mut Keys) -> (u64, bool) { - if let Some(&(key, settled, _)) = keys.get(&Rc::as_ptr(node)) { - return (key, settled); - } - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - // Exact whatever its inputs (an exact stand-in among them). - if node - .guarantee - .as_ref() - .is_some_and(|guarantee| guarantee.is_exact()) - { - "exact".hash(&mut hasher); - if node.result_kind == OperatorResultKind::State { - Rc::as_ptr(node).hash(&mut hasher); - } - let known = (hasher.finish(), true); - keys.insert(Rc::as_ptr(node), (known.0, known.1, node.clone())); - return known; - } - let spec = node.guarantee.as_ref().map(|guarantee| &guarantee.spec); - format!("{spec:?}").hash(&mut hasher); - let state = node.result_kind == OperatorResultKind::State; - let known = match &node.guarantee { - Some(guarantee) if !state => ( - hasher.finish(), - guarantee.spec.error.evaluate().is_some() - && guarantee.spec.failure_probability.evaluate().is_some(), - ), - _ => { - let children: Vec<(u64, bool)> = node - .children() - .into_iter() - .map(|child| key_and_settled(child, keys)) - .collect(); - let settled = node.guarantee.is_none() && children.iter().all(|(_, settled)| *settled); - if !(settled && !state) { - node.operator.kind_name().hash(&mut hasher); - state.hash(&mut hasher); - for (key, _) in &children { - key.hash(&mut hasher); - } - } - (hasher.finish(), settled) - } - }; - keys.insert(Rc::as_ptr(node), (known.0, known.1, node.clone())); - known -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::groups::tests::{candidate, catalog, reader_of, relative, root, values_node}; - use asap_logical_asap_optimization::test_support::entry_with; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - /// The depth of each output of `built`. - fn depths(built: &Built) -> Vec { - fn depth(node: &OperatorNode) -> usize { - 1 + node - .children() - .into_iter() - .map(|child| depth(child)) - .max() - .unwrap_or(0) - } - built - .as_ref() - .expect("built") - .iter() - .map(|node| depth(node)) - .collect() - } - - // A two-candidate link cycle above an approximate member: `A` reads - // the leaf or `B`; `B` reads `A` or a second leaf. Walking from the - // top enters the cycle at `A`, so `B` is built there with `A` cut off - // (a cycle read), over the second leaf only. Such outputs are not - // kept: a later walk that starts at `B` builds it in full, over `A`, - // as with no kept builds, for one member and for a pair. - #[test] - fn outputs_built_across_a_cycle_are_not_kept() { - let approximate_leaf = |value: f64, error: f64| { - Rc::new( - (*values_node(value)) - .clone() - .with_guarantee(Some(relative(error, 0.01))), - ) - }; - let leaf = approximate_leaf(1.0, 0.01); - let other_leaf = approximate_leaf(2.0, 0.02); - let a = reader_of(&leaf); - let b = reader_of(&leaf); - let top = reader_of(&a); - let physical = catalog( - vec![(leaf, 0), (a, 1), (b, 2), (top, 3), (other_leaf, 4)], - vec![ - candidate(0, &[], false), - candidate(1, &[&[0, 2]], false), - candidate(2, &[&[1, 4]], false), - candidate(3, &[&[1]], true), - candidate(4, &[], false), - ], - vec![root(3)], - ); - let producers: Vec>> = physical - .candidates - .iter() - .map(|candidate| { - candidate - .producer_candidates_per_input - .iter() - .map(|producers| producers.iter().copied().collect()) - .collect() - }) - .collect(); - // Leaves 0 and 4 stand for approximate candidates. - let below: Vec> = vec![ - BTreeSet::from([0]), - BTreeSet::from([0, 4]), - BTreeSet::from([0, 4]), - BTreeSet::from([0, 4]), - BTreeSet::from([4]), - ]; - let exact_stand_ins: Vec>> = (0..physical.candidates.len()) - .map(|_| OnceCell::new()) - .collect(); - let entry = entry_with( - AccuracyTarget::max_error(0.01).failure_probability(0.01), - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))), - ); - let data = DataWorkload::default(); - let cost_inputs = asap_cost::CostInputs::default(); - let models = SelectionModels { - accuracy: &asap_accuracy::BuiltinAccuracyModel, - evidence: &asap_accuracy::NoAccuracyEvidence, - cost_inputs: &cost_inputs, - }; - let output_candidates = vec![vec![PhysicalCandidateId(3)]]; - let checker = Checker { - physical: &physical, - exact_stand_ins: &exact_stand_ins, - producer_candidates_per_input: &producers, - output_candidates: &output_candidates, - below: &below, - query: 0, - root: &physical.query_roots[0], - references: &[0], - entry: &entry, - data: &data, - models: &models, - }; - for approximate in [&[0][..], &[0, 4]] { - let mut kept = BuiltBefore::default(); - // From the top: `B` is built with `A` cut off, over the second - // leaf alone (depth 2). - let mut walk = Walk::default(); - checker - .built(3, approximate, &mut walk, &mut kept) - .expect("built"); - assert!(walk.cycle_reads > 0); - assert!(walk.built[&2].across_cycle); - assert_eq!(depths(&walk.built[&2].outputs)[0], 2, "{approximate:?}"); - // From `B`: in full, its first output reads `A` over the leaf - // (depth 3), with kept builds as with none. - let from_b = checker.built(2, approximate, &mut Walk::default(), &mut kept); - let fresh = checker.built( - 2, - approximate, - &mut Walk::default(), - &mut BuiltBefore::default(), - ); - assert_eq!(depths(&from_b), depths(&fresh), "{approximate:?}"); - assert_eq!(depths(&fresh)[0], 3, "{approximate:?}"); - } - } -} diff --git a/crates/plan-selection/src/groups/tests.rs b/crates/plan-selection/src/groups/tests.rs deleted file mode 100644 index f9c2ce4ee..000000000 --- a/crates/plan-selection/src/groups/tests.rs +++ /dev/null @@ -1,1085 +0,0 @@ -use std::rc::Rc; - -use super::*; -use asap_types::ir::properties::{AccuracyMetric, BoundExpr, ProbabilityExpr}; -use asap_types::ir::scalar::expr_ir::ScalarValue; -use asap_types::ir::schema::SketchAlgorithm; -use asap_types::ir::schema::{DataType, Field}; -use asap_types::ir::{QueryRoot, ScalarExpr}; -use asap_types::physical::Medium; -use asap_types::stages::{ - CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, - OperatorNodeOfCandidateForMemoGroupId, PhysicalCandidate, ProducerCandidates, -}; - -// --------------------------------------------------------------------------- -// Hand-built catalogs -// --------------------------------------------------------------------------- - -/// A one-row `VALUES` node of `value`. -pub(super) fn values_node(value: f64) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(value))]], - schema: Schema::lifted(vec![Field::plain("value", DataType::Float64, false)], None), - })) - .unwrap() -} - -/// A `VALUES` node of `rows` one-column rows. -fn rows_node(rows: usize) -> Rc { - let schema = Schema::lifted(vec![Field::plain("value", DataType::Float64, false)], None); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(1.0))]; rows], - schema, - })) - .unwrap() -} - -/// A stateless one-input reader of `child`. -pub(super) fn reader_of(child: &Rc) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Dedup { - cols: Vec::new(), - child: Rc::clone(child), - })) - .unwrap() -} - -/// A catalog of `nodes`, node `i` in memo group `nodes[i].1` (each node its -/// own candidate of that group, all implementation roots), with -/// `candidates` and one query root per entry of `roots`. -pub(super) fn catalog( - nodes: Vec<(Rc, u32)>, - candidates: Vec, - roots: Vec>, -) -> PhysicalASAPMemoCandidates { - let mut index_in_group: HashMap = HashMap::new(); - PhysicalASAPMemoCandidates::new( - nodes - .into_iter() - .map(|(physical_node, group)| { - let index = index_in_group.entry(group).or_default(); - *index += 1; - OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![CandidateForMemoGroupId { - group: MemoGroupId(group), - index: *index - 1, - }], - is_implementation_root: true, - accuracy_budget_parts: 1, - physical_node, - } - }) - .collect(), - candidates, - roots, - Vec::new(), - ) -} - -/// One candidate of operator node `node` at query time, not materialized, -/// reading `producers` per input, that may produce a query's output when -/// `output`. -pub(super) fn candidate(node: usize, producers: &[&[usize]], output: bool) -> PhysicalCandidate { - PhysicalCandidate { - operator_node: OperatorNodeOfCandidateForMemoGroupId(node), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - producer_candidates_per_input: producers - .iter() - .map(|input| { - input - .iter() - .map(|&id| PhysicalCandidateId(id)) - .collect::() - }) - .collect(), - may_produce_query_output: output, - may_keep_across_evaluations_per_sole_reader_query: Vec::new(), - } -} - -pub(super) fn root(group: u32) -> QueryRoot { - QueryRoot::Operator(MemoGroupId(group)) -} - -fn prepared(physical: &PhysicalASAPMemoCandidates) -> MilpInputs { - prepare_milp_inputs( - physical, - &[], - &DataWorkload::default(), - &CostInputs::default(), - ) -} - -fn id(candidate: usize) -> PhysicalCandidateId { - PhysicalCandidateId(candidate) -} - -// --------------------------------------------------------------------------- -// Dominance -// --------------------------------------------------------------------------- - -fn usage() -> CandidateUsage { - CandidateUsage { - cpu_ops_per_execution: 10.0, - scanned_bytes_per_execution: 10.0, - work_ms_per_execution: 10.0, - own_latency_ms: 10.0, - cost_per_execution: Some(1.0), - standing_cost_per_second: Some(1.0), - average_memory_bytes: 10.0, - peak_memory_bytes: 10, - disk_bytes: 10, - held_bytes: 10, - raw_data_kept_bytes: 10, - maintained_window_bytes: 10, - output_rows: 10, - output_bytes: 10, - } -} - -fn data_with(usage: CandidateUsage) -> CandidateData { - CandidateData { - approximate: true, - accuracy_budget_parts: 1, - usage: Ok(usage), - ..CandidateData::default() - } -} - -pub(super) fn relative(error: f64, failure_probability: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { value: error }, - ProbabilityExpr::Constant { - value: failure_probability, - }, - Vec::new(), - ) -} - -// Candidate dominance needs every condition: a candidate with equal keys, a -// guarantee at least as strong and no larger number dominates; changing any one condition stops it. -#[test] -fn dominance_needs_every_condition() { - let producers = vec![vec![PhysicalCandidateId(7)]]; - let other_producers = vec![vec![PhysicalCandidateId(8)]]; - let keep = vec![false]; - let other_keep = vec![true]; - let schema = rows_node(1).schema.clone(); - let other_schema = Schema::lifted(vec![Field::plain("v", DataType::Int64, false)], None); - let strong = relative(0.01, 0.01); - let weak = relative(0.02, 0.01); - let values_kind = - OperatorKindForReaders::Other(std::mem::discriminant(&rows_node(1).operator), "Values"); - let key = DominanceKey { - group: MemoGroupId(3), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - shareable: false, - producer_candidates_per_input: &producers, - readers: vec![(PhysicalCandidateId(9), 0)], - output_references: vec![0], - reader_constraint_ids: vec![1], - may_keep_across_evaluations_per_sole_reader_query: &keep, - output_schema: &schema, - operator_kind: values_kind, - }; - let profile = |guarantee| DominanceProfile { - same: key.clone(), - guarantee, - }; - let better = profile(Some(&strong)); - let worse = profile(Some(&weak)); - let mut worse_usage = usage(); - worse_usage.cpu_ops_per_execution = 20.0; - let (better_data, worse_data) = (data_with(usage()), data_with(worse_usage)); - assert!(dominates((&better, &better_data), (&worse, &worse_data))); - assert!(!dominates((&worse, &worse_data), (&better, &better_data))); - - // Each key field. - let changed_keys = [ - ( - "group", - DominanceKey { - group: MemoGroupId(4), - ..key.clone() - }, - ), - ( - "timing", - DominanceKey { - timing: ExecutionTiming::IngestionTime, - ..key.clone() - }, - ), - ( - "materialization", - DominanceKey { - materialization: Materialization::MatAtQuery(Medium::Memory), - ..key.clone() - }, - ), - ( - "shareable", - DominanceKey { - shareable: true, - ..key.clone() - }, - ), - ( - "producers", - DominanceKey { - producer_candidates_per_input: &other_producers, - ..key.clone() - }, - ), - ( - "readers", - DominanceKey { - readers: vec![(PhysicalCandidateId(10), 0)], - ..key.clone() - }, - ), - ( - "outputs", - DominanceKey { - output_references: vec![1], - ..key.clone() - }, - ), - ( - "reader constraints", - DominanceKey { - reader_constraint_ids: vec![2], - ..key.clone() - }, - ), - ( - "keep flags", - DominanceKey { - may_keep_across_evaluations_per_sole_reader_query: &other_keep, - ..key.clone() - }, - ), - ( - "schema", - DominanceKey { - output_schema: &other_schema, - ..key.clone() - }, - ), - ( - "operator kind", - DominanceKey { - operator_kind: OperatorKindForReaders::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 5, - }, - ..key.clone() - }, - ), - ]; - for (name, changed) in changed_keys { - let by = DominanceProfile { - same: changed, - ..better.clone() - }; - assert!( - !dominates((&by, &better_data), (&worse, &worse_data)), - "{name}" - ); - } - // (approx, parts), guarantee. - let mut data = better_data.clone(); - data.approximate = false; - assert!( - !dominates((&better, &data), (&worse, &worse_data)), - "approx" - ); - let mut data = better_data.clone(); - data.accuracy_budget_parts = 2; - assert!(!dominates((&better, &data), (&worse, &worse_data)), "parts"); - let by = profile(Some(&weak)); - let class = profile(Some(&strong)); - assert!( - !dominates((&by, &better_data), (&class, &worse_data)), - "guarantee" - ); - // Each number. - for number in 0..compared_numbers(&usage()).len() { - let mut larger = usage(); - match number { - 0 => larger.cpu_ops_per_execution = 30.0, - 1 => larger.scanned_bytes_per_execution = 30.0, - 2 => larger.work_ms_per_execution = 30.0, - 3 => larger.own_latency_ms = 30.0, - 4 => larger.average_memory_bytes = 30.0, - 5 => larger.peak_memory_bytes = 30, - 6 => larger.disk_bytes = 30, - 7 => larger.held_bytes = 30, - 8 => larger.raw_data_kept_bytes = 30, - 9 => larger.maintained_window_bytes = 30, - 10 => larger.output_rows = 30, - _ => larger.output_bytes = 30, - } - assert!( - !dominates((&better, &data_with(larger)), (&worse, &worse_data)), - "number {number}" - ); - } - // A plan-dependent or uncosted usage is never compared. - let mut data = better_data.clone(); - data.usage_depends_on = vec![UsageContext::ProducerOutput]; - assert!( - !dominates((&better, &data), (&worse, &worse_data)), - "context" - ); - let mut data = better_data.clone(); - data.usage = Err("uncosted".into()); - assert!(!dominates((&better, &data), (&worse, &worse_data)), "error"); -} - -// A guarantee is at least as strong when equal, exact, or no larger in one -// metric and scope; different metrics and unknown bounds do not compare. -#[test] -fn guarantee_strength_compares_constant_bounds_in_one_metric() { - let strong = relative(0.01, 0.01); - let weak = relative(0.02, 0.02); - let exact = AccuracyGuarantee::exact("exact"); - let rank = AccuracyGuarantee::new( - AccuracyMetric::Rank, - BoundExpr::Constant { value: 0.001 }, - ProbabilityExpr::Zero, - Vec::new(), - ); - assert!(guarantee_at_least_as_strong(Some(&strong), Some(&weak))); - assert!(!guarantee_at_least_as_strong(Some(&weak), Some(&strong))); - assert!(guarantee_at_least_as_strong(Some(&exact), Some(&weak))); - assert!(guarantee_at_least_as_strong(None, None)); - assert!(!guarantee_at_least_as_strong(Some(&strong), None)); - assert!(!guarantee_at_least_as_strong(Some(&rank), Some(&weak))); -} - -// Among dominated candidates the dominating one is reported; of two equal -// candidates the smaller id is kept; a candidate better in one number and worse in -// another is kept. -#[test] -fn pruning_keeps_the_smallest_equal_class_and_incomparable_ones() { - let producers = vec![vec![PhysicalCandidateId(9)]]; - let keep = Vec::new(); - let schema = rows_node(1).schema.clone(); - let key = DominanceKey { - group: MemoGroupId(0), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - shareable: false, - producer_candidates_per_input: &producers, - readers: Vec::new(), - output_references: vec![0], - reader_constraint_ids: Vec::new(), - may_keep_across_evaluations_per_sole_reader_query: &keep, - output_schema: &schema, - operator_kind: OperatorKindForReaders::Other( - std::mem::discriminant(&rows_node(1).operator), - "Values", - ), - }; - let profiles = vec![ - DominanceProfile { - same: key.clone(), - guarantee: None, - }; - 4 - ]; - let mut worse = usage(); - worse.output_rows = 20; - let mut incomparable = usage(); - incomparable.cpu_ops_per_execution = 5.0; - incomparable.held_bytes = 20; - let data = vec![ - data_with(usage()), - data_with(usage()), - data_with(worse), - data_with(incomparable), - ]; - let pruned = prune_dominated_candidates(&profiles, &data); - let summary: Vec<_> = pruned - .iter() - .map(|pruned| (pruned.class.0, pruned.dominated_by.0)) - .collect(); - assert_eq!(summary, vec![(1, 0), (2, 0)]); -} - -// Through `prepare_milp_inputs`: of two deduplications in one memo group -// over one leaf, read by one reader and costing the same, the later one is -// deleted with its links, and the reader reads the other. -#[test] -fn a_dominated_class_is_deleted_with_its_links() { - let schema = Schema::lifted( - vec![ - Field::plain("a", DataType::Float64, false), - Field::plain("b", DataType::Float64, false), - ], - None, - ); - let leaf = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(1.0)); 2]; 4], - schema, - })) - .unwrap(); - let dedup = |cols: Vec| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Dedup { - cols, - child: Rc::clone(&leaf), - })) - .unwrap() - }; - // Two equal nodes: equal schemas and costs. - let (first, second) = (dedup(vec![0]), dedup(vec![0])); - let reader = reader_of(&first); - let physical = catalog( - vec![(leaf, 0), (first, 1), (second, 1), (reader, 2)], - vec![ - candidate(0, &[], false), - candidate(1, &[&[0]], false), - candidate(2, &[&[0]], false), - candidate(3, &[&[1, 2]], true), - ], - vec![root(2)], - ); - let inputs = prepared(&physical); - let (leaf, kept, deleted, top) = (id(0), id(1), id(2), id(3)); - assert_eq!( - inputs.candidate_data[kept.0].usage, - inputs.candidate_data[deleted.0].usage - ); - assert_eq!( - inputs - .pruned_candidates - .iter() - .map(|pruned| (pruned.class, pruned.dominated_by)) - .collect::>(), - vec![(deleted, kept)] - ); - assert_eq!( - inputs.producer_candidates_per_input[top.0], - vec![vec![kept]] - ); - assert!(inputs.producer_candidates_per_input[deleted.0].is_empty()); - assert_eq!(inputs.contexts[0].options, vec![leaf, kept, top]); -} - -// --------------------------------------------------------------------------- -// Execution contexts -// --------------------------------------------------------------------------- - -/// A leaf, a materialized (shareable) middle, and two queries each reading -/// the middle through its own stateless top; a third query reads only its -/// own leaf. Candidate `i` runs operator node `i`. -fn shared_middle() -> PhysicalASAPMemoCandidates { - let leaf = values_node(1.0); - let middle = reader_of(&leaf); - let top_0 = reader_of(&middle); - let top_1 = reader_of(&middle); - let lone = values_node(2.0); - let candidates = vec![ - candidate(0, &[], false), - PhysicalCandidate { - materialization: Materialization::MatAtQuery(Medium::Memory), - ..candidate(1, &[&[0]], false) - }, - candidate(2, &[&[1]], true), - candidate(3, &[&[1]], true), - candidate(4, &[], true), - ]; - catalog( - vec![(leaf, 0), (middle, 1), (top_0, 2), (top_1, 3), (lone, 4)], - candidates, - vec![root(2), root(3), root(4)], - ) -} - -// Each query context holds its own top and the shared middle as a read -// across its boundary; the middle's own context holds the leaf below it; -// no `I_r` is read, so there is none; a query that cannot reach the middle holds only its -// own candidate. -#[test] -fn execution_contexts_on_a_hand_built_catalog() { - let physical = shared_middle(); - let inputs = prepared(&physical); - let (leaf, middle, top_0, top_1, lone) = (id(0), id(1), id(2), id(3), id(4)); - assert!(CandidateLinks::new(&physical).is_shareable(middle)); - assert!(inputs.pruned_candidates.is_empty()); - let contexts: Vec<(ContextKind, Vec)> = inputs - .contexts - .iter() - .map(|context| (context.kind, context.options.clone())) - .collect(); - assert_eq!( - contexts, - vec![ - (ContextKind::Query(0), vec![middle, top_0]), - (ContextKind::Query(1), vec![middle, top_1]), - (ContextKind::Query(2), vec![lone]), - (ContextKind::Shared(middle), vec![leaf]), - ] - ); - assert_eq!(inputs.context_of_shareable, BTreeMap::from([(middle, 3)])); - assert!(inputs.contexts[3].computes(RunKind::PerContext)); - assert!(!inputs.contexts[3].computes(RunKind::Shareable)); - assert_eq!( - inputs.output_candidates, - vec![vec![top_0], vec![top_1], vec![lone]] - ); -} - -/// Stage 2's output for `queries` and what Stage 3 also reads. -struct ThroughStages1And2 { - physical: PhysicalASAPMemoCandidates, - entries: Vec, - data: DataWorkload, -} - -/// `queries` (PromQL over metric `m`, each predictable and repeating every -/// minute with a 1 % error target, over metric `m` with 100 series) -/// through Stage 1 and Stage 2 as the facade runs them, over the complete pane-width domain, with raw data and -/// query-time results retained. -fn through_stages_1_and_2(queries: &[&str]) -> ThroughStages1And2 { - use asap_logical_asap_optimization::test_support::lower_promql; - let roots: Vec = queries - .iter() - .map(|query| QueryRoot::Operator(lower_promql(query, target()))) - .collect(); - through_stages_1_and_2_of(roots) -} - -/// The accuracy target of [`through_stages_1_and_2`]'s queries. -fn target() -> asap_types::ir::properties::AccuracyTarget { - asap_types::ir::properties::AccuracyTarget::max_error(0.01).failure_probability(0.01) -} - -/// [`through_stages_1_and_2`] for query roots `roots`. -fn through_stages_1_and_2_of(roots: Vec) -> ThroughStages1And2 { - use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; - use asap_logical_asap_optimization::generate_per_computation_logical_memo_candidates; - use asap_logical_asap_optimization::test_support::entry_with; - use asap_types::workload::{ - DurationMs, Predictability, QueryRecurrence, RepeatedDemand, RepetitionInterval, - }; - let target = target(); - let every_minute = - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))); - let entries: Vec<_> = roots - .iter() - .map(|_| QueryWorkloadEntry { - predictability: Predictability::Predictable { known_at: None }, - ..entry_with(target.clone(), every_minute.clone()) - }) - .collect(); - let mut data = DataWorkload::sampled_every(["m".to_string()], DurationMs(1_000)); - let source = data.sources.get_mut("m").expect("m"); - source.series_count = Some(100); - source.values_non_negative = true; - let targets = vec![target.clone(); roots.len()]; - let pass1 = generate_per_computation_logical_memo_candidates( - asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - &targets, - &asap_accuracy::BuiltinAccuracyModel, - &asap_accuracy::NoAccuracyEvidence, - &data, - None, - ) - .unwrap(); - let logical = generate_logical_asap_memo_candidates(pass1, &entries).unwrap(); - let physical = asap_physical_asap_optimization::generate_physical_asap_memo_candidates( - &logical, &entries, &data, true, true, - ) - .unwrap(); - // Stage 2 copies each node's `accuracy_budget_parts` from its memo - // candidates, the largest for a factored node. - for node in &physical.operator_nodes_of_candidates_for_memo_groups { - let largest = node - .candidates_for_memo_group - .iter() - .map(|id| logical.memo.candidates[id.group.0 as usize][id.index].accuracy_budget_parts) - .max(); - assert_eq!(Some(node.accuracy_budget_parts), largest); - } - ThroughStages1And2 { - physical, - entries, - data, - } -} - -// Through the real Stages 1 and 2: two quantile queries over 1m and 5m -// windows of one metric can share a pane chain. Both queries reach it, it -// is an option of both query contexts with a context of its own, and every -// context computes only what runs there. -// No quotient: every approximate candidate weighs 1. -#[test] -fn two_queries_share_a_pane_chain_through_the_real_pipeline() { - let stages = through_stages_1_and_2(&[ - "quantile_over_time(0.9, m[15m])", - "quantile_over_time(0.9, m[1h])", - ]); - let physical = &stages.physical; - let links = CandidateLinks::new(physical); - let inputs = prepare_milp_inputs( - physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - let shared_chains: Vec = (0..physical.candidates.len()) - .map(PhysicalCandidateId) - .filter(|candidate| { - matches!( - physical.physical_node(*candidate).asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && links.is_shareable(*candidate) - && inputs.contexts[0].options.contains(candidate) - && inputs.contexts[1].options.contains(candidate) - }) - .collect(); - assert!(!shared_chains.is_empty()); - for &chain in &shared_chains { - let context = inputs.context_of_shareable[&chain]; - assert_eq!(inputs.contexts[context].kind, ContextKind::Shared(chain)); - } - // Every context computes only candidates of its own kind and reads the - // others across its boundary. - for context in &inputs.contexts { - for &option in &context.options { - let kind = inputs.kind[option.0]; - assert!( - context.computes(kind) - || kind == RunKind::Shareable - || (kind == RunKind::Ingestion - && !matches!(context.kind, ContextKind::Ingestion(_))) - ); - } - } - for weights in &inputs.accuracy_weights { - assert!(!weights.is_empty()); - assert!( - weights - .values() - .all(|&weight| weight == AccuracyWeight::One), - "{weights:?}" - ); - } -} - -// --------------------------------------------------------------------------- -// Accuracy weights -// --------------------------------------------------------------------------- - -/// The sketch algorithm and `parts` of each approximate candidate of -/// `physical`, with its weight for query 0 (`None`: no weight). -fn sketch_weights( - physical: &PhysicalASAPMemoCandidates, - inputs: &MilpInputs, -) -> Vec<(SketchAlgorithm, usize, Option)> { - (0..physical.candidates.len()) - .filter(|&class| inputs.candidate_data[class].approximate) - .map(|class| { - let id = PhysicalCandidateId(class); - let algorithm = match physical.physical_node(id).asap() { - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => *kind.algorithm(), - _ => unreachable!("an approximate candidate is a sketch"), - }; - ( - algorithm, - inputs.candidate_data[class].accuracy_budget_parts, - inputs.accuracy_weights[0].get(&id).copied(), - ) - }) - .collect() -} - -// `q0.5 / q0.9` through the real stages, weights from the composition: a -// KLL rank error has no rule through a quotient, so Pass 1 prunes every KLL -// candidate (#1129); a DDSketch relative -// error composes, but a lone denominator sized for the whole target -// (`parts = 1`) misses it (`ε/(1 − ε) > ε`) while one sized for two parts -// meets it. -#[test] -fn ratio_weights_come_from_the_composition() { - let stages = through_stages_1_and_2(&[ - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - ]); - let inputs = prepare_milp_inputs( - &stages.physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - let weights = sketch_weights(&stages.physical, &inputs); - let of = |algorithm: SketchAlgorithm, parts: usize| -> BTreeSet> { - weights - .iter() - .filter(|(a, p, _)| *a == algorithm && *p == parts) - .map(|(_, _, weight)| *weight) - .collect() - }; - let missed = Some(AccuracyWeight::Unusable( - UnusableReason::AccuracyTargetMissed, - )); - // Pass 1 prunes every KLL: its rank error composes no guarantee - // through the quotient. - for parts in [1, 2] { - assert_eq!( - of(SketchAlgorithm::Kll, parts), - BTreeSet::new(), - "{weights:?}" - ); - } - // At one part: numerators meet the target, denominators miss it. - assert_eq!( - of(SketchAlgorithm::DDSketch, 1), - BTreeSet::from([Some(AccuracyWeight::One), missed]), - "{weights:?}" - ); - assert_eq!( - of(SketchAlgorithm::DDSketch, 2), - BTreeSet::from([Some(AccuracyWeight::One)]), - "{weights:?}" - ); - // Two DDSketches at two parts compose: no pair is excluded. - assert!(inputs.accuracy_excluded_pairs[0].is_empty()); -} - -// RangeHeavy T8+T5's shape: `q0.9 / q0.5` with both operands also queried -// alone. One DDSketch pane chain sized for two shares of the budget serves -// both quotient operands and both lone quantiles, as the old Stage 3 -// engine planned it: the check accepts the quotient over it (`(α + α)/(1 − α) ≤ ε` at -// `α = 1 − (1 + ε)^{−1/2}`), so it weighs 1 for every query. A chain sized -// for the whole budget (`α = ε`) never serves the quotient's denominator. -#[test] -fn one_pane_chain_serves_a_ratio_and_its_operands() { - let stages = through_stages_1_and_2(&[ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])", - "quantile_over_time(0.9, m[5m])", - "quantile_over_time(0.5, m[5m])", - ]); - let inputs = prepare_milp_inputs( - &stages.physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - let alpha = |class: usize| match stages - .physical - .physical_node(PhysicalCandidateId(class)) - .asap() - { - Some(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => match kind.params() { - asap_types::ir::schema::SketchParams::DDSketch { alpha } => Some(*alpha), - _ => None, - }, - _ => None, - }; - let weight = |query: usize, class: usize| { - inputs.accuracy_weights[query] - .get(&PhysicalCandidateId(class)) - .copied() - }; - let two_shares = 1.0 - 1.01f64.powf(-0.5); - let shared: Vec = (0..stages.physical.candidates.len()) - .filter(|&class| alpha(class).is_some()) - .filter(|&class| (0..3).all(|query| weight(query, class) == Some(AccuracyWeight::One))) - .collect(); - assert!(!shared.is_empty()); - for &class in &shared { - assert!( - (alpha(class).unwrap() - two_shares).abs() < 1e-9, - "{:?}", - alpha(class) - ); - } - // A chain sized for the whole budget weighs 1 for the ratio only where - // its values are the numerator's alone, so it never also serves both - // lone quantiles. - for class in (0..stages.physical.candidates.len()).filter(|&class| alpha(class) == Some(0.01)) { - assert!( - !(0..3).all(|query| weight(query, class) == Some(AccuracyWeight::One)), - "{class}" - ); - } -} - -// The checks keep what they build for later checks and skip repeated -// exact stand-ins: on RangeHeavy T8+T5's shape (a quotient and both -// operands alone), on a quotient of three quantiles (where pairs are -// excluded) and on a ratio with one approximate operand, the weights and -// excluded pairs equal those with no kept builds and no skipped stand-in. -#[test] -fn kept_builds_give_the_weights_of_fresh_builds() { - kept_builds_match_fresh_builds(&[ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])", - "quantile_over_time(0.9, m[5m])", - "quantile_over_time(0.5, m[5m])", - ]); - let three = kept_builds_match_fresh_builds(&[ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m]) / quantile_over_time(0.1, m[5m])", - ]); - assert!(!three.accuracy_excluded_pairs[0].is_empty()); - // A scalar ratio of two output references whose approximate values sit - // under one operand only: the other reference's output candidates - // reach no member, and its first exact stand-in must still be built, - // or the ratio has no root to check and every approximate candidate - // would weigh nothing. - use asap_logical_asap_optimization::test_support::lower_promql; - use asap_types::ir::scalar::{ArithmeticOpKind, ExprSemantics}; - let operand = |query: &str| { - Box::new(ScalarExpr::PromqlScalarFromVector(lower_promql( - query, - target(), - ))) - }; - let ratio = QueryRoot::Scalar(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Div, - left: operand("avg_over_time(m[5m])"), - right: operand("quantile_over_time(0.5, m[5m])"), - semantics: ExprSemantics::Promql, - }); - let ratio = kept_builds_match_fresh_builds_of(vec![ratio]); - assert!(!ratio.accuracy_weights[0].is_empty()); -} - -/// Asserts that `queries`' accuracy weights and excluded pairs are the -/// same with kept builds and with none ([`accuracy_positions::FRESH_BUILDS`]); -/// returns the former. -fn kept_builds_match_fresh_builds(queries: &[&str]) -> MilpInputs { - use asap_logical_asap_optimization::test_support::lower_promql; - kept_builds_match_fresh_builds_of( - queries - .iter() - .map(|query| QueryRoot::Operator(lower_promql(query, target()))) - .collect(), - ) -} - -/// [`kept_builds_match_fresh_builds`] for query roots `roots`. -fn kept_builds_match_fresh_builds_of(roots: Vec) -> MilpInputs { - let stages = through_stages_1_and_2_of(roots); - let prepare = |fresh: bool| { - accuracy_positions::FRESH_BUILDS.with(|cell| cell.set(fresh)); - let inputs = prepare_milp_inputs( - &stages.physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - accuracy_positions::FRESH_BUILDS.with(|cell| cell.set(false)); - inputs - }; - let (kept, fresh) = (prepare(false), prepare(true)); - assert!(kept - .accuracy_weights - .iter() - .any(|weights| !weights.is_empty())); - assert_eq!(kept.accuracy_weights, fresh.accuracy_weights); - assert_eq!(kept.accuracy_excluded_pairs, fresh.accuracy_excluded_pairs); - kept -} - -// --------------------------------------------------------------------------- -// Ingestion pipelines per refresh window -// --------------------------------------------------------------------------- - -/// Ingestion-time views of metric `m`, each read by its own query: -/// - candidates 2 and 3: a KLL and a DDSketch over `TimeRange(10 s)` (one -/// refresh window, (10 s, 0)); -/// - 5: a KLL over `TimeRange(60 s)`, window (60 s, 0); -/// - 8: a KLL over `TimeRange(10 s)` over `TimeShift(5 s)`, window -/// (10 s, 5 s), through two `NotMaterialized` ingestion-time nodes; -/// - 9: a KLL over the whole scan, an unbounded window. -/// -/// Candidates 0, 1, 4, 6 and 7 (the scan, ranges and shift) are -/// `NotMaterialized` at ingestion time; 10–14 are the query-time readers. -fn ingestion_views_of_three_windows_and_an_unbounded_one() -> PhysicalASAPMemoCandidates { - use asap_types::ir::operator::{Reduction, Source, TimeShift}; - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::{GroupingStrategy, SketchKind, SketchParams, SummaryUpdate}; - use asap_types::ir::TimeRangeKind; - use std::time::Duration; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap(); - let range = |seconds: u64, child: &Rc| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(seconds), - kind: TimeRangeKind::Range, - child: Rc::clone(child), - })) - .unwrap() - }; - let sketch = |algorithm: SketchAlgorithm, child: &Rc| { - let params = match algorithm { - SketchAlgorithm::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - _ => SketchParams::DDSketch { alpha: 0.01 }, - }; - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() - }; - let range_10s = range(10, &scan); - let range_60s = range(60, &scan); - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 5_000, - at: None, - }, - child: Rc::clone(&scan), - })) - .unwrap(); - let range_10s_shifted = range(10, &shifted); - let views = [ - sketch(SketchAlgorithm::Kll, &range_10s), - sketch(SketchAlgorithm::DDSketch, &range_10s), - sketch(SketchAlgorithm::Kll, &range_60s), - sketch(SketchAlgorithm::Kll, &range_10s_shifted), - sketch(SketchAlgorithm::Kll, &scan), - ]; - let mut nodes = vec![ - (scan, 0), - (range_10s, 1), - (views[0].clone(), 2), - (views[1].clone(), 3), - (range_60s, 4), - (views[2].clone(), 5), - (shifted, 6), - (range_10s_shifted, 7), - (views[3].clone(), 8), - (views[4].clone(), 9), - ]; - let at_ingestion = |node: usize, producers: &[&[usize]], view: bool| PhysicalCandidate { - timing: ExecutionTiming::IngestionTime, - materialization: if view { - Materialization::MatAtIngestion(Medium::Memory) - } else { - Materialization::NotMaterialized - }, - ..candidate(node, producers, false) - }; - let mut candidates = vec![ - at_ingestion(0, &[], false), - at_ingestion(1, &[&[0]], false), - at_ingestion(2, &[&[1]], true), - at_ingestion(3, &[&[1]], true), - at_ingestion(4, &[&[0]], false), - at_ingestion(5, &[&[4]], true), - at_ingestion(6, &[&[0]], false), - at_ingestion(7, &[&[6]], false), - at_ingestion(8, &[&[7]], true), - at_ingestion(9, &[&[0]], true), - ]; - let mut roots = Vec::new(); - for (index, view) in [2, 3, 5, 8, 9].into_iter().enumerate() { - let group = 10 + index as u32; - nodes.push((reader_of(&nodes[view].0), group)); - candidates.push(candidate(10 + index, &[&[view]], true)); - roots.push(root(group)); - } - catalog(nodes, candidates, roots) -} - -// One `I_r` per refresh window: the two views over `TimeRange(10 s)` share -// one; the 60 s view and the shifted 10 s view (window (10 s, 5 s), through -// a `NotMaterialized` shift and range) have their own; the unbounded view -// gets an `Alone` one. Each `I_r` holds the scan and the stateless nodes -// below its views, and each view's context reads across into its `I_r`. -#[test] -fn execution_contexts_have_one_ingestion_pipeline_per_refresh_window() { - let physical = ingestion_views_of_three_windows_and_an_unbounded_one(); - let links = CandidateLinks::new(&physical); - let alive = vec![true; physical.candidates.len()]; - let windows = refresh_windows(&physical, &links); - let window = |width_ms, phase_ms| RefreshWindow { width_ms, phase_ms }; - assert_eq!( - windows, - HashMap::from([ - (id(2), Some(window(10_000, 0))), - (id(3), Some(window(10_000, 0))), - (id(5), Some(window(60_000, 0))), - (id(8), Some(window(10_000, 5_000))), - (id(9), None), - ]) - ); - let contexts = execution_contexts( - &physical, - &links.kind, - &producer_candidates_left(&links, &alive), - &output_candidates(&physical, &alive), - &windows, - |_| true, - ); - let ingestion: Vec<(ContextKind, Vec)> = contexts.contexts[5..9] - .iter() - .map(|context| (context.kind, context.options.clone())) - .collect(); - let pipeline = |width_ms, phase_ms| { - ContextKind::Ingestion(IngestionPipeline::Window(window(width_ms, phase_ms))) - }; - assert_eq!( - ingestion, - vec![ - (pipeline(10_000, 0), vec![id(0), id(1)]), - (pipeline(10_000, 5_000), vec![id(0), id(6), id(7)]), - (pipeline(60_000, 0), vec![id(0), id(4)]), - ( - ContextKind::Ingestion(IngestionPipeline::Alone(id(9))), - vec![id(0)] - ), - ] - ); - assert_eq!( - contexts.ingestion_context_of_shareable, - BTreeMap::from([(id(2), 5), (id(3), 5), (id(5), 7), (id(8), 6), (id(9), 8)]) - ); - let shared = contexts.context_of_shareable[&id(8)]; - assert_eq!(contexts.contexts[shared].options, vec![id(7)]); -} - -// Every producer candidate of every input reads one view window, the -// assumption behind computing a refresh window from first producers. -#[test] -fn producers_of_an_input_read_one_view_window() { - let physical = ingestion_views_of_three_windows_and_an_unbounded_one(); - assert!(producer_window_disagreements_of_all(&physical).is_empty()); -} diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 77100f7df..d6c5fef6f 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -1,5858 +1,23 @@ -//! #509 Stage 3, plan selection (MVP): the only stage that computes cost. -//! -//! Each CandidatePhysicalASAPDAG from Stage 2 is checked against every -//! query's accuracy target: the query's output meets it when its root -//! guarantee does (theoretical: Stage 1 composes it with the built-in -//! analytical rules, and this stage binds the statistics it left symbolic -//! from the data workload and the deployment's operand-domain contracts), -//! or when the deployment's empirical evidence shows this -//! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, -//! read only here). The deployment's accuracy model may instead accept the -//! output directly, whatever its guarantee says -//! (`AccuracyModel::accepts_output_regardless_of_guarantee`); the -//! accept-all model the facade passes to ignore accuracy guarantees -//! (`asap_accuracy::IgnoreAccuracyGuarantees`) accepts every output so, and -//! the report records each one the guarantee check would have rejected. Before that, for -//! a query with a target, Count-Min is admitted only over weights proven -//! non-negative, which the executor needs to run it; a query without one -//! constrains nothing. -//! A miss rejects the candidate as invalid, with a reason; only this stage -//! discards valid candidates. -//! A candidate whose evaluation of some query takes longer than that -//! query's latency bound, or than its repeat interval, is rejected as -//! invalid too ("Latency" below). -//! Every valid candidate's resource usage is measured node by node over its -//! physical DAG ([`ResourceUsage`](asap_cost::ResourceUsage)), and the deployment's [`CostModel`](asap_cost::CostModel) -//! chooses among them ("Cost models" below). The rest are reported valid -//! but costlier (or slower, or larger). -//! The group selection MILP ([`select_plan`], #1095) selects one candidate -//! per memo group and execution context without constructing the Cartesian -//! product of local candidates; it prices cost model 1 only (#888). -//! -//! ## Cost model: cost per second of wall time -//! -//! A candidate's cost is what running it costs per second ([`COST_UNIT`](asap_cost::COST_UNIT)), -//! so queries with different recurrences add up. A node here is one -//! operator node of the candidate's physical plan DAG (`PhysicalASAPDAG`), -//! not a machine. Each node's price for one -//! computation is multiplied by how often it is computed per second -//! ([`NodeCost::cost_per_second`](asap_cost::NodeCost::cost_per_second)): -//! -//! * A `NotMaterialized` node is recomputed for each consumer that reads it -//! and is not stored: it is charged once per consumer -//! ([`NodeCost::multiplier`](asap_cost::NodeCost::multiplier)) at the sum of their rates. Its consumers are -//! the queries whose sub-DAG reaches it and the materialized nodes above -//! it, counted without looking through a materialized node: a node below -//! one is computed when that node is. The `MatAtIngestion` consumers of -//! one refresh window (`PhysicalASAPDAG::refresh_window`: for a tumbling window -//! chain, its tumbling-window width) are refreshed in one ingestion run and count as -//! one consumer: a `Scan` under a KLL chain and a DDSketch chain of 10 s -//! tumbling windows is computed once per second of arriving data, under chains of -//! 10 s and 60 s tumbling windows twice. Stage 2 keeps a shared node one -//! node (it copies it only for readers at two timings). A shared -//! `NotMaterialized` input costs what reading it separately does (#509 -//! Example 4) only when every consumer reads the same window: a query-time `Scan` is priced over the longest window -//! read above it, so each consumer pays for that window, and a scan a 1 m -//! and a 5 m window share costs more than two separate scans. Stage 2 -//! offers the highest shared node built once per evaluation (`MatAtQuery`) -//! as another candidate. -//! * A `MatAtQuery` node is computed once for all the queries that reach -//! it, at their combined rate: queries repeating at the same interval are -//! evaluated together, so each distinct interval counts once. -//! * A `MatAtIngestion` node runs as data arrives: it is priced over one -//! second of arriving rows, once per second, whatever its readers' rates. -//! A tumbling-window chain maintained this way updates only its newest -//! tumbling window: each arriving row is placed in it and updates it. -//! * A query repeating every `I` ms runs `1000 / I` times per second, and an -//! estimated rate runs at that rate. One-off, scheduled and unknown -//! queries are spread over the settings' `non_repeating_query_horizon_seconds` -//! (default 3600 s): `invocations / horizon` per second. -//! -//! Under cost model 1, memory is charged as bytes held per second -//! ([`ResourceUsage::average_memory_bytes`](asap_cost::ResourceUsage::average_memory_bytes)), at -//! `cost_per_memory_byte_second` per byte per second. A `MatAtIngestion` -//! node keeps its state every second: twice for a whole window (the window -//! being built and the completed one queries read) and `N + 1` tumbling windows for a -//! chain of `N` (the `N` the query merges and the one being built). A -//! `MatAtQuery` node's state is reused only within one evaluation, so it is -//! held only while that evaluation runs: its bytes times the latency of the -//! longest query reading it, once per evaluation. A query-time -//! `NotMaterialized` node's working memory (a sketch being built, an exact -//! aggregate's values) is charged once per consumer that recomputes it: -//! for a query, its bytes times that query's latency, at its rate; for a -//! materialized node reading it (directly or through other -//! `NotMaterialized` nodes), its bytes times the latency of the longest -//! query reading that node, at the node's rate, as a `MatAtQuery` state is. -//! A `MatAtQueryKeptAcrossEvaluations` tumbling-window chain (#509 Example -//! 4's B3, offered by Stage 2 when the deployment keeps query-time results, -//! `DeploymentCapabilities::query_time_results_retained`) is kept across -//! evaluations: each evaluation scans one tumbling window of raw data and builds the -//! newest tumbling window, and the `N − 1` older tumbling windows, which complete the readers' -//! look-back, are kept every second (in memory, all `N` at the peak). When -//! Stage 2 offers both query-time options for a chain, plan selection -//! prices each and the cheaper wins. -//! -//! A query whose workload entry declares a range evaluation -//! ([`RangeEvaluation`](asap_types::workload::RangeEvaluation): each request evaluates it at every `step_ms` over -//! the last `span_ms`, as a dashboard's `query_range` does) is priced as -//! `S = span_ms / step_ms + 1` evaluations per request (61 for a 1 h span -//! at 1 min steps): -//! -//! * every query-time computation that is not kept across evaluations -//! (`NotMaterialized`, `MatAtQuery`) runs once per step, `S` times per -//! request; a `MatAtIngestion` state is maintained as before, and the -//! merge and estimate over it run once per step; -//! * a chain kept across evaluations is built once per request: Stage 2 -//! offers it only when the requests repeat at its tumbling-window width, so in the -//! steady state each request builds its newest tumbling window and every other tumbling window -//! any of its steps reads was built by an earlier request or step; -//! * a tumbling-window chain kept beyond one evaluation (ingestion time or kept -//! across evaluations) retains its input's offset plus its window plus -//! the longest `span_ms` among its readers (`RealizedPlan::tumbling_window_retention_ms`), -//! so the earliest step of the next request still finds its tumbling windows: that -//! many more tumbling windows are kept every second. -//! -//! Latency stays per evaluation (one step). Without a range evaluation -//! `S = 1` and the retention is the window, so every price is as above. -//! Memory a computation holds only while it runs counts in -//! [`ResourceUsage::peak_memory_bytes`](asap_cost::ResourceUsage::peak_memory_bytes), which cost models 2 and 3 read, and -//! under cost model 1 for the time it is held, as above. (A `NotMaterialized` -//! node computed at ingestion time is a streaming operator holding one row, -//! and is not charged.) -//! -//! Prices come from `asap-cost`'s analytical model (`analytical::estimate_operator`) over edge -//! statistics derived from the [`DataWorkload`](asap_types::workload::DataWorkload) and a fixed default group -//! count; summary build and estimation are priced as rows × sketch depth and -//! groups × k. A query-time scan reads the longest window above it (a time -//! shift moves a window without widening it); a time shift is free, and a -//! time range is priced by the rows it keeps. A tumbling-window chain is -//! priced as one build over its whole window plus one operation per row to -//! place it in its tumbling window, and keeps one state per group and tumbling window; a merge -//! costs one operation per state it reads. Kept state is sized by its bytes: a sketch by its parameters, a -//! fixed-size exact accumulator (sum, count, min, max, rate, ...) as one -//! value per group, and exact state that grows with the data (quantile, -//! top-k, distinct count, L2, entropy keep every value or item) as one value -//! per input row of its group, assuming rows spread evenly over groups and, -//! for distinct items, that every row may be distinct. These numbers are -//! illustrative, not calibrated: a binary, join or concat node is priced from -//! its first input only (MVP; follow-up). -//! A cost error on any candidate aborts selection; it is reachable only from -//! bad declared rates or bad [`CostInputs`](asap_cost::CostInputs). -//! -//! ## Measured operation costs -//! -//! A deployment whose runtime measured its own costs gives them per -//! operator ([`CostInputs::measured_operation_costs`](asap_cost::CostInputs::measured_operation_costs), -//! [`MeasuredOperationCosts`](asap_cost::MeasuredOperationCosts)): for an operator kind at ingestion or query -//! time (and, for a summary operator, its sketch or exact accumulator; for -//! an aggregate, each measure), CPU ns per unit and per computation. A node -//! with an entry takes `units × ns_per_unit + ns_per_invocation` per -//! computation, its scanning included; the units are what the operator -//! processes ([`OperatorKind`](asap_cost::OperatorKind): samples a scan reads, rows a build inserts, -//! states a merge reads, groups an estimate reads, ...). A query-time scan -//! is the fetch and decode of raw data samples from the database holding them, -//! so a plan that reads raw data pays for it. A sketch whose parameters -//! no entry has exactly is priced between the two measured configurations -//! of the same sketch nearest in state size, on a log scale, or from the -//! nearest one outside their range. A kept state's bytes are the entry's -//! measured `memory_bytes_per_state` when it has one. The newest window's -//! compaction (see "Latency") merges with the state's query-time merge -//! entry too, and each query request pays the table's `ns_per_evaluation` -//! once, on the node it reads first and in its latency. -//! -//! A node without an entry is priced from the formulas above, its CPU -//! operations times one calibration factor: the measured time of every -//! node the table prices, over all candidates, divided by what the -//! formulas give for those nodes. Without it a table that measures some -//! operators (a slow sketch) and not others (the exact plan's scan and -//! aggregate) makes the unmeasured plan look cheap only because its nodes -//! keep the formulas' scale. The factor is one global heuristic, not a -//! measurement: an unmeasured operator whose real cost is far from the -//! measured ones' ratio is still mispriced, so a partial table can still -//! mis-rank plans (e.g. a cheap raw scan and exact quantile priced at the -//! slow sketch's ratio make the exact rebuild look costlier than it is). -//! Only a table covering every operator kind the candidates use makes the -//! choice the measured one. Each node reports how it was priced -//! ([`NodePriceSource`](asap_cost::NodePriceSource)). -//! Measured times count as CPU operations of [`ResourceSpeed`](asap_cost::ResourceSpeed), so latency -//! and all three cost models read them like any other operation. -//! -//! ## Cost models -//! -//! [`CostInputs::cost_model`](asap_cost::CostInputs::cost_model) (the cost inputs [`select_plan`] receives) chooses how -//! usage becomes a choice; the default is cost model 1 with -//! [`ResourceUnitCost::illustrative`](asap_cost::ResourceUnitCost::illustrative). -//! [`select_plan`] searches under cost model 1 only; cost models 2 and 3 -//! are an `UnsupportedCostModel` error until #888 (the whole-plan reference -//! pricer still ranks plans under all three): -//! -//! * [`CostModel::UsageOverTime`](asap_cost::CostModel::UsageOverTime) (cost model 1, pay-as-you-go): each -//! node's CPU operations, scanned bytes and memory byte-seconds per second -//! (and disk bytes, unpriced by default) times their prices; the cheapest -//! candidate wins. -//! * [`CostModel::ProvisionedCapacity`](asap_cost::CostModel::ProvisionedCapacity) (cost model 2): the candidate's peak -//! memory, peak CPU cores and disk times their prices per second; the -//! cheapest wins. Peak memory assumes every evaluation overlaps: kept -//! in-memory state plus each computation's working memory, once per -//! consumer that recomputes it. Peak cores are the ingestion work's cores -//! plus one per query evaluating at once. -//! * [`CostModel::ClusterCapacity`](asap_cost::CostModel::ClusterCapacity) (cost model 3, own cluster): no prices. -//! A candidate over the cluster's memory, disk or cores is invalid, and -//! [`ClusterObjective`](asap_cost::ClusterObjective) chooses among the rest (lowest worst-query -//! latency, or least peak memory). -//! -//! Disk holds state materialized on `Medium::Disk` (Stage 2 places none -//! there yet) and the raw data query-time scans read: per source, the -//! longest window a scan reads. A deployment that does not retain raw data -//! (`DeploymentCapabilities::raw_data_retained`) keeps no raw data samples -//! after they arrive, so Stage 2 generates no query-time scan of raw data. -//! -//! ## Latency -//! -//! Following the sketch-bench RQE cost model (§5, batch latency), a query's -//! latency per evaluation is its longest chain of work, each step on a -//! full core of elastic CPU (no queueing): the compaction of the newest -//! window of an ingestion-time state it reads, then its own query-time -//! work. Its own work is the sum, over every query-time node it reaches -//! (shared ones included, each once), of one computation's CPU operations -//! and scanned bytes timed by [`ResourceSpeed`](asap_cost::ResourceSpeed)'s `ms_per_cpu_op` and -//! `ms_per_scanned_byte`. A state whose ingestion work needs `k` cores keeps -//! `k` partial copies of its newest window, and compacting them takes -//! `(k − 1)` merges for each of its states per window, each one operation -//! (times the calibration factor), or the state's measured merge time when -//! the measured table has one; with -//! one core there is nothing to compact. A query with -//! `Some(bound)` whose latency is above the -//! bound makes the candidate invalid, with the latency as the reason -//! ([`RejectionCause::LatencyBoundExceeded`]). With or without a bound, a -//! query repeating every `I` ms whose latency is above `I` makes the -//! candidate invalid too ([`RejectionCause::LatencyOverRepeatInterval`]): -//! each evaluation starts before the last one ends, so it falls further -//! behind with every evaluation. Only a fixed interval is checked: an -//! estimated rate, a schedule or a one-time query has none. A range -//! evaluation is checked by its latency per step, as priced: the latency -//! model gives each step a full core, so a request's steps may run in -//! parallel. Run one after another they take `S` times as long, which this -//! check does not assume. -//! [`ResourceUsage::latency_per_query`](asap_cost::ResourceUsage::latency_per_query) reports every query's latency. -//! -//! No memory limit applies outside cost model 3, where the cluster's memory -//! is declared ([`ClusterLimits::memory_bytes`](asap_cost::ClusterLimits::memory_bytes)); without one, a selected -//! plan whose peak memory is over `LARGE_PEAK_MEMORY_BYTES` (10^12) gets a -//! warning, which `asap_planner` reports ([`large_peak_memory_warning`]). - -#[cfg(test)] -use asap_types::workload::RepeatedDemand; +//! #509 Stage 3, plan selection (concepts doc +//! `memo-groups-properties-requirements.md`, section 3, "Stage 3: baseline +//! selection and realization", and sections 4 and 5). +//! +//! [`select_plan`] reads Stage 2's physical candidates $\mathcal C$ and the +//! admissible part of the physical edges $\mathcal E$, selects with a MILP +//! over $x_v$, $y_{u,c,i}$, $z_{q,v}$ and $t_v$ (constraints (A)–(I); +//! (G) and (H) hold because $y$ exists only on admissible edges), and +//! builds the plan (`PhysicalASAPDAG`) from $\{x = 1\}$ and $\{y = 1\}$ +//! as its last step. The extensions add execution contexts $\kappa$ with $x^{\mathrm{mat}}_v$, +//! $x_{\kappa,v}$, $y_{\kappa,u,c,i}$ and the retention amount $h_v$ +//! (section 5), and accuracy at each query root, constraint (K), with +//! per-candidate error bounds $E_c$ and $D_c$ (section 4). Costs come from +//! `asap-cost`. -mod accuracy_binding; -mod checks; -mod models; -// #1095 §2.0: the group selection MILP's inputs (dominance, execution -// contexts, accuracy weights). -mod groups; -// #1095 §2.0: the group selection MILP and the decode of its solution. mod milp; mod report; -// Stage 3's entry, `select_plan`, over the group selection MILP (#1095). mod select; -// The whole-plan selector behind `reference_pricer` and the tests. -#[cfg(any(test, feature = "reference-pricer"))] -mod selector; -pub use models::AccuracyCheckInputs; -pub(crate) use models::SelectionModels; + pub use report::{ - large_peak_memory_warning, AcceptedAccuracy, ErrorSelectingPlan, InexactObjective, - RejectionCause, SearchReport, SearchStatus, Selection, + large_peak_memory_warning, AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, }; pub use select::select_plan; -#[cfg(test)] -pub(crate) use selector::select_among_plans; - -/// The whole-workload reference pricer (feature `reference-pricer`): the -/// selector Stage 3 used before the memo search, over whole physical plans -/// a developer tool enumerates (`asap_devtools::memo_pricing`, -/// `planner_fuzz`). Not part of Stage 3's interface. -#[cfg(feature = "reference-pricer")] -pub mod reference_pricer { - pub use crate::report::{Rejection, WholePlanSelection}; - use crate::{AccuracyCheckInputs, ErrorSelectingPlan, SelectionModels}; - use asap_cost::CostInputs; - use asap_types::physical::PhysicalWorkloadPlan; - use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - - /// Check, price and rank every plan of `candidates`, keeping every - /// candidate: the selected one, each valid one's cost and accepted - /// accuracy, and why each other one lost. - /// - /// Note: with measured operation costs and no frozen calibration factor, - /// the formulas are calibrated over `candidates` alone, so costs are not - /// on the scale of a facade run's selection (`--price-enumerated`). - pub fn select_among_plans( - candidates: &[PhysicalWorkloadPlan], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - accuracy_checks: &AccuracyCheckInputs<'_>, - cost_inputs: &CostInputs, - ) -> Result { - crate::selector::select_among_plans( - candidates, - entries, - data, - SelectionModels::new(accuracy_checks, cost_inputs), - ) - } -} - -/// `roots` sized as Pass 1 sizes them for `targets` over `data` -/// ([`with_summaries_sized_for_targets`](asap_accuracy::sizing::with_summaries_sized_for_targets)). -#[cfg(test)] -pub(crate) fn sized_as_pass1( - roots: &[asap_types::ir::QueryRoot], - targets: &[asap_types::ir::properties::AccuracyTarget], - data: &asap_types::workload::DataWorkload, -) -> Vec { - asap_accuracy::sizing::with_summaries_sized_for_targets(roots, targets, data) - .expect("sized summaries rebuild") -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::accuracy_binding::bound_root_guarantee; - use crate::accuracy_binding::{bind_statistics, workload_statistic}; - use crate::checks::guarantee_check; - use crate::checks::{accuracy_check, CheckFailure}; - use crate::report::WholePlanSelection; - use crate::selector::PlanSelector; - use asap_accuracy::composition::INPUT_ROW_COUNT; - use asap_accuracy::estimators::hll; - use asap_accuracy::propagation::root_guarantee; - use asap_cost::analytical::AnalyticalCostError; - use asap_cost::execution_demand::evaluation_rate; - use asap_cost::state_bytes::{sketch_shape, summary_shape, StateInputRows}; - use asap_cost::{kept_tumbling_window_chains, node_map, plan_cost, NodeMap}; - use asap_cost::{lookup, StateFamily}; - use asap_cost::{ - CalibrationFit, ClusterLimits, ClusterObjective, CostInputs, CostModel, NodePriceSource, - OperatorKind, PlanCost, ResourceUsage, COST_SOURCE, MEASURED_COST_SOURCE, - MIXED_COST_SOURCE, - }; - use asap_cost::{MeasuredOperationCosts, OperatorCost, ResourceUnitCost}; - use asap_types::ir::operator::AggIntent; - use asap_types::ir::physical_export::{ - PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, - }; - use asap_types::ir::properties::{AccuracyGuarantee, AccuracyScope, ExecutionTiming}; - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::schema::DataType; - use asap_types::ir::schema::{ - FieldDataType, Schema, SketchAlgorithm, SketchParams, WeightDomain, - }; - use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; - use asap_types::physical::PhysicalWorkloadPlan; - use asap_types::physical::{Materialization, Medium}; - use asap_types::workload::bounds::with_confirmed_weights; - use asap_types::workload::bounds::{declared_series, series_read, values_per_result}; - use asap_types::workload::{ - DataWorkload, QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, - }; - use std::collections::{BTreeMap, BTreeSet}; - use std::rc::Rc; - - /// [`plan_cost`] of one candidate alone, its calibration factor fitted - /// over its own measured nodes (as [`select_plan`] fits it over all). - fn self_calibrated_cost( - dag: &PhysicalASAPDAG, - nodes: &NodeMap<'_>, - kept_chains: &std::collections::HashSet, - recurrence: &[QueryRecurrence], - range_evaluations: &[Option], - data: &DataWorkload, - settings: &CostInputs, - ) -> Result { - let at = |formula_scale| { - plan_cost( - dag, - nodes, - kept_chains, - recurrence, - range_evaluations, - data, - settings, - formula_scale, - ) - }; - let (_, fit) = at(1.0)?; - Ok(at(fit.factor())?.0) - } - - /// [`self_calibrated_cost`] with `dag`'s node map and kept tumbling-window chains. - fn price( - dag: &PhysicalASAPDAG, - recurrence: &[QueryRecurrence], - data: &DataWorkload, - settings: &CostInputs, - ) -> Result { - self_calibrated_cost( - dag, - &node_map(dag), - &kept_tumbling_window_chains(dag), - recurrence, - &[], - data, - settings, - ) - } - use asap_accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, - }; - use asap_logical_asap_optimization::test_support::{entry, entry_with, lower_promql}; - use asap_logical_asap_optimization::DeploymentCapabilities; - use asap_physical_asap_optimization::optimize_physical_asap; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::properties::{AccuracyMetric, BoundExpr, ProbabilityExpr}; - use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; - use asap_types::ir::{ExprSemantics, ScalarExpr}; - use asap_types::workload::{ - DataArrival, LabelSetDistinctValues, PrometheusMetricType, SourceFacts, - }; - - /// `G` of `root` over `data` ([`values_per_result`]). - fn values_of(root: &QueryRoot, data: &DataWorkload) -> Option { - values_per_result(root, series_read(root, data), data) - } - - /// [`bound_root_guarantee`] of `root` over `data`. - fn bound_of( - root: &QueryRoot, - data: &DataWorkload, - models: &SelectionModels<'_>, - ) -> Option { - bound_root_guarantee(root, series_read(root, data), data, models) - } - - /// `n` queries repeating every 10 s, as Example 1's panels do. - fn recurrences(n: usize) -> Vec { - vec![repeating(10_000); n] - } - - /// A query repeating every `interval_ms`. - fn repeating(interval_ms: u32) -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - asap_types::workload::RepetitionInterval(interval_ms), - )) - } - - /// The metrics and tables the tests' queries read. - const TEST_SOURCES: [&str; 5] = ["m", "x", "a", "b", "t"]; - - /// Data arriving at 10 000 rows per second, each test source holding - /// 10 000 series. - fn data() -> DataWorkload { - with_series(Some(10_000)) - } - - /// [`data`] with every test source holding `series` (`None`: unknown). - fn with_series(series: Option) -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 10_000.0, - }, - sources: TEST_SOURCES - .iter() - .map(|source| { - let facts = SourceFacts { - series_count: series, - ..SourceFacts::default() - }; - (source.to_string(), facts) - }) - .collect(), - } - } - - /// Explicit domain assumption needed for interpolated DDS quantiles. - fn non_negative_data() -> DataWorkload { - let mut data = data(); - for facts in data.sources.values_mut() { - facts.values_non_negative = true; - } - data - } - - /// `data` with each of `types`' metrics declared of its type. - fn with_metric_types( - mut data: DataWorkload, - types: &[(&str, PrometheusMetricType)], - ) -> DataWorkload { - for (metric, metric_type) in types { - data.sources - .entry(metric.to_string()) - .or_default() - .metric_type = Some(*metric_type); - } - data - } - - /// Classic HLL's failure probability over a source binds only that - /// source's distinct bound: a bound keyed to another source leaves it - /// unknown. - #[test] - fn classic_hll_failure_probability_binds_only_its_own_sources_bound() { - let statistic = asap_logical_asap_optimization::test_support::hll_classic_failure_statistic( - 15, 0.05, "a", - ); - let bounded = |source: &str| { - let mut data = data(); - data.sources - .get_mut(source) - .unwrap() - .max_distinct_per_evaluation = Some(100); - data - }; - assert_eq!(workload_statistic(&statistic, None, &bounded("b")), None); - assert_eq!( - workload_statistic(&statistic, None, &bounded("a")), - Some(hll::classic_failure_probability(15, 0.05, 100)) - ); - assert!(hll::classic_failure_probability(15, 0.05, 100) <= 0.01); - } - - /// Exact (P1), whole-expression CMS+heap (P2) and CountSketch+heap (P3) realizations of - /// one approximate top-k query, in Pass 1 catalog order. - fn candidates() -> Vec { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let root = lower_promql("topk by (job) (10, sum_over_time(m[1m]))", target.clone()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let topk = inventory - .targets - .iter() - .position(|t| t.choice_count() > 2) - .unwrap(); - [0, 1, 2] - .into_iter() - .map(|index| { - let mut choice = vec![0; inventory.targets.len()]; - choice[topk] = index; - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - let roots = sized_as_pass1(&roots, std::slice::from_ref(&target), &data()); - let mut candidate = optimize_physical_asap("L", &roots, &[], &data(), false) - .unwrap() - .remove(0); - candidate.id = format!("P{}", index + 1); - candidate - }) - .collect() - } - - /// A summary whose analytical guarantee misses the target is rejected as - /// invalid with a reason; the exact plan is then selected. (Top-K - /// estimates carry no guarantee since #575, so a quantile summary is used.) - #[test] - fn accuracy_failing_candidate_is_rejected_as_invalid() { - let planned = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates("quantile_over_time(0.99, m[1m])", &planned); - let strict = AccuracyTarget::max_error(1e-6).failure_probability(1e-9); - let summaries: Vec<_> = candidates - .iter() - .filter(|c| has_summary_estimate(c)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!summaries.is_empty()); - let selection = select_among_plans( - &candidates, - &[entry_with(strict, repeating(10_000))], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - assert!(!summaries.contains(&selection.selected.as_str())); - for id in &summaries { - let rejection = selection.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, RejectionCause::AccuracyTargetMissed); - assert!( - rejection.reason.contains("misses target"), - "{}", - rejection.reason - ); - assert!(!selection.costs.contains_key(*id)); - } - } - - /// For a query with a target, Count-Min over weights not proven - /// non-negative is invalid. - #[test] - fn count_min_over_signed_weights_is_rejected_as_invalid() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let selection = select_among_plans( - &candidates(), - &[entry_with( - target - .clone() - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1), - repeating(10_000), - )], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!( - p2.cause, - RejectionCause::CountMinWeightsNotProvenNonNegative - ); - assert!(p2.reason.contains("non-negative"), "{}", p2.reason); - } - - /// Count-Min over raw data samples of `m` is admitted only when the data - /// workload declares `m` a counter: a gauge, an undeclared metric or a - /// counter under another name leaves the requirement unmet. Admitted, - /// the top-k still needs evidence about its output. - #[test] - fn count_min_over_raw_data_samples_needs_a_declared_counter() { - use asap_types::workload::PrometheusMetricType; - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let with_types = |types: &[(&str, PrometheusMetricType)]| with_metric_types(data(), types); - for (types, admitted) in [ - (vec![("m", PrometheusMetricType::Counter)], true), - (vec![("m", PrometheusMetricType::Gauge)], false), - (vec![("m_total", PrometheusMetricType::Counter)], false), - (vec![], false), - ] { - let data = with_types(&types); - let selection = select_among_plans( - &candidates(), - &[entry_with( - target - .clone() - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1), - repeating(10_000), - )], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - // Admitted weights leave the top-k output without a guarantee, - // which the built-in model does not accept; unproven weights - // reject the plan before any accuracy check. - let expected_cause = if admitted { - RejectionCause::NoAccuracyGuarantee - } else { - RejectionCause::CountMinWeightsNotProvenNonNegative - }; - assert_eq!(p2.cause, expected_cause); - assert_eq!( - !p2.reason.contains("non-negative"), - admitted, - "{types:?}: {}", - p2.reason - ); - if admitted { - assert!(p2.reason.contains("no guarantee"), "{}", p2.reason); - } else { - assert!( - p2.reason.contains("not declared a counter"), - "{}", - p2.reason - ); - } - } - } - - /// The Count-Min sketch algorithms (plain or with a heap) `candidate` - /// builds. - fn count_min_builds(candidate: &PhysicalWorkloadPlan) -> Vec { - candidate - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if matches!( - kind.algorithm(), - SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap - ) => - { - Some(*kind.algorithm()) - } - _ => None, - }) - .collect() - } - - /// `increase` (and `rate`, `irate`) over a counter is non-negative, since - /// resets are corrected: the Count-Min-with-heap top-k of - /// `topk(10, sum by (job) (increase(data_total[5m])))` passes the weight - /// check when `data_total` is declared a counter, and goes on to the - /// accuracy and cost checks. Over a gauge it is still rejected for its - /// weights (issue #773). - #[test] - fn count_min_over_a_counter_derivative_needs_a_declared_counter() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - // An `offset` moves the window, not the samples' sign. - for query in [ - "topk(10, sum by (job) (increase(data_total[5m])))", - "topk(10, sum by (job) (rate(data_total[5m])))", - "topk(10, sum by (job) (irate(data_total[5m])))", - "topk(10, sum by (job) (rate(data_total[5m] offset 1h)))", - ] { - let candidates = all_candidates(query, &target); - let count_min: Vec<&str> = candidates - .iter() - .filter(|c| count_min_builds(c).contains(&SketchAlgorithm::CmsWithHeap)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!count_min.is_empty(), "{query}"); - for (metric_type, admitted) in [ - (PrometheusMetricType::Counter, true), - (PrometheusMetricType::Gauge, false), - ] { - let data = with_metric_types(data(), &[("data_total", metric_type)]); - let rejected = match select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) { - Ok(selection) => selection.rejected, - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => rejected, - Err(error) => panic!("{query}: {error}"), - }; - for id in &count_min { - let weights_rejected = rejected.iter().any(|r| { - &r.id == id - && r.cause == RejectionCause::CountMinWeightsNotProvenNonNegative - }); - assert_eq!( - weights_rejected, !admitted, - "{query} over a {metric_type:?}, {id}: {rejected:?}" - ); - } - } - } - } - - /// The `_sum` series of a declared histogram family is a counter: with - /// only the family `latency` declared a histogram, the Count-Min-with-heap - /// top-k of `topk(10, sum by (job) (increase(latency_sum[5m])))` passes - /// the weight check. Undeclared, or with `latency` a gauge, it does not - /// (issue #1165). - #[test] - fn count_min_over_a_declared_histograms_sum_series_is_admitted() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let query = "topk(10, sum by (job) (increase(latency_sum[5m])))"; - let candidates = all_candidates(query, &target); - let count_min: Vec<&str> = candidates - .iter() - .filter(|c| count_min_builds(c).contains(&SketchAlgorithm::CmsWithHeap)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!count_min.is_empty()); - for (family_type, admitted) in [ - (Some(PrometheusMetricType::Histogram), true), - (Some(PrometheusMetricType::Gauge), false), - (None, false), - ] { - let mut data = data(); - data.sources.insert( - "latency".into(), - SourceFacts { - series_count: Some(10_000), - metric_type: family_type, - ..SourceFacts::default() - }, - ); - let rejected = match select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) { - Ok(selection) => selection.rejected, - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => rejected, - Err(error) => panic!("{family_type:?}: {error}"), - }; - for id in &count_min { - let weights_rejected = rejected.iter().any(|r| { - &r.id == id && r.cause == RejectionCause::CountMinWeightsNotProvenNonNegative - }); - assert_eq!( - weights_rejected, !admitted, - "latency declared {family_type:?}, {id}: {rejected:?}" - ); - } - } - } - - /// A gauge declared non-negative - /// ([`SourceFacts::values_non_negative`]) proves its raw data samples, and - /// plain sums of them, non-negative: the Count-Min-with-heap top-k of - /// `topk by (job) (10, sum_over_time(m[1m]))` (the ASAPQuery workloads' - /// top-k-sum shape) passes the weight check, and the selected plan carries - /// [`NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples`]. A rate or - /// increase of that gauge stays unproven: a reset correction over a - /// gauge can still give a negative value. - #[test] - fn count_min_over_raw_data_samples_of_a_gauge_declared_non_negative_is_admitted() { - use asap_types::ir::schema::NonNegativeWeightProof; - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let mut gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - gauge.sources.get_mut("m").unwrap().values_non_negative = true; - for (query, admitted) in [ - ("topk by (job) (10, sum_over_time(m[1m]))", true), - ("topk(10, sum by (job) (increase(m[5m])))", false), - ] { - let candidates = all_candidates(query, &target); - let count_min: Vec<&PhysicalWorkloadPlan> = candidates - .iter() - .filter(|c| count_min_builds(c).contains(&SketchAlgorithm::CmsWithHeap)) - .collect(); - assert!(!count_min.is_empty(), "{query}"); - let rejected = match select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &gauge, - SelectionModels::builtin(), - ) { - Ok(selection) => selection.rejected, - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => rejected, - Err(error) => panic!("{query}: {error}"), - }; - for candidate in &count_min { - let weights_rejected = rejected.iter().any(|r| { - r.id == candidate.id - && r.cause == RejectionCause::CountMinWeightsNotProvenNonNegative - }); - assert_eq!( - weights_rejected, !admitted, - "{query}, {}: {rejected:?}", - candidate.id - ); - if !admitted { - continue; - } - // The proof the executor reads. - let proofs: Vec = with_confirmed_weights(&candidate.roots, &gauge) - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|n| match n.asap() { - Some(ASAPOp::SummaryAgg { - input, - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::CmsWithHeap => { - Some(input.weight_domain.clone()) - } - _ => None, - }) - .collect(); - assert!(!proofs.is_empty(), "{}", candidate.id); - assert!( - proofs.iter().all(|domain| *domain - == WeightDomain::NonNegative { - proof: NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples - }), - "{}: {proofs:?}", - candidate.id - ); - } - } - } - - /// The sign gate is an indicator that only 1 (proven) opens: any other - /// value a binder supplies, 0 included, leaves it unbound rather than - /// reading as a perfect bound. Other statistics pass through. - #[test] - fn the_sign_gate_opens_only_at_one() { - use asap_accuracy::test_support::raw_data_samples_sign_statistic; - let gated = AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Product { - factors: vec![ - BoundExpr::Constant { value: 0.01 }, - BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic("m"), - }, - ], - }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - for value in [0.0, 0.5, 2.0, -1.0] { - assert_eq!( - bind_statistics(&gated, &|_| Some(value)) - .spec - .error - .evaluate(), - None, - "{value}" - ); - } - assert_eq!( - bind_statistics(&gated, &|_| Some(1.0)) - .spec - .error - .evaluate(), - Some(0.01) - ); - let mut norm = gated; - norm.spec.error = BoundExpr::Unknown { - statistic: INPUT_ROW_COUNT.into(), - }; - assert_eq!( - bind_statistics(&norm, &|_| Some(0.0)).spec.error.evaluate(), - Some(0.0) - ); - } - - /// A relative bound gated on the sign of one metric's samples is bound - /// when the data workload declares that metric a counter, and stays - /// unknown for a gauge, an undeclared metric or a counter by another - /// name. - #[test] - fn sign_of_counter_raw_data_samples_is_bound_from_the_declared_type() { - use asap_accuracy::test_support::raw_data_samples_sign_statistic; - let gated = AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Product { - factors: vec![ - BoundExpr::Constant { value: 0.01 }, - BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic("m"), - }, - ], - }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - let node = lower_promql("sum(m)", AccuracyTarget::exact()); - let root = QueryRoot::Operator(Rc::new((*node).clone().with_guarantee(Some(gated)))); - let with = |types: &[(&str, PrometheusMetricType)]| with_metric_types(data(), types); - let bound = |data: &DataWorkload| { - bound_of(&root, data, &SelectionModels::builtin()) - .unwrap() - .spec - .error - .evaluate() - }; - assert_eq!( - bound(&with(&[("m", PrometheusMetricType::Counter)])), - Some(0.01) - ); - for types in [ - vec![("m", PrometheusMetricType::Gauge)], - vec![("m_total", PrometheusMetricType::Counter)], - vec![], - ] { - assert_eq!(bound(&with(&types)), None, "{types:?}"); - } - } - - /// Every consumer of a run sees the selected plan with its weight - /// requirements answered: the logical plan (`selected_logical`) and the - /// Offered one at a time and keeping only the best so far, the selector - /// selects what `select_among_plans` selects over every candidate at - /// once, at the same cost, under every cost model. - #[test] - fn streaming_selection_selects_what_select_plan_selects() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let root = lower_promql("quantile_over_time(0.99, m[5m])", target.clone()); - let entries = [entry_with(target, repeating(60_000))]; - let data = with_series(Some(10_000)); - let candidates = &exhaustive_candidates(&[QueryRoot::Operator(root)], &entries, &data); - assert!(candidates.len() > 2); - let cluster = |objective| { - CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective, - }) - }; - for cost_model in [ - CostModel::UsageOverTime(ResourceUnitCost::illustrative()), - CostModel::ProvisionedCapacity(ResourceUnitCost::illustrative()), - cluster(ClusterObjective::LowestLatency), - cluster(ClusterObjective::LeastMemory), - ] { - let settings = CostInputs { - cost_model, - ..CostInputs::default() - }; - let models = SelectionModels::builtin().with_cost_inputs(&settings); - let all = select_among_plans(candidates, &entries, &data, models).unwrap(); - let mut selector = PlanSelector::new(&entries, &data, models, false).unwrap(); - for candidate in candidates { - selector.offer(candidate).unwrap(); - } - let (streamed, built) = selector.finish().unwrap(); - assert_eq!(streamed.selected, all.selected); - assert_eq!(built.id, all.selected); - assert_eq!(streamed.plan, all.plan); - assert_eq!(streamed.costs[&streamed.selected], all.costs[&all.selected]); - assert_eq!(streamed.candidates_checked, candidates.len() as u64); - // Only the selected candidate is kept. - assert_eq!(streamed.costs.len(), 1); - } - } - - /// The selected plan's open requirement becomes a proof when confirmed - /// and stays unproven otherwise; the executor reads no data workload. - #[test] - fn confirmed_weights_rewrite_only_the_summary_update() { - use asap_types::ir::schema::NonNegativeWeightProof; - use asap_types::workload::PrometheusMetricType; - let p2 = &candidates()[1]; - let domains = |roots: &[QueryRoot]| -> Vec { - roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|n| match n.asap() { - Some(ASAPOp::SummaryAgg { input, .. }) => Some(input.weight_domain.clone()), - _ => None, - }) - .collect() - }; - assert_eq!( - domains(&p2.roots), - [WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }] - ); - let counter = with_metric_types(data(), &[("m", PrometheusMetricType::Counter)]); - let confirmed = with_confirmed_weights(&p2.roots, &counter); - assert_eq!( - domains(&confirmed), - [WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples - }] - ); - let mut gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - gauge.sources.get_mut("m").unwrap().values_non_negative = true; - assert_eq!( - domains(&with_confirmed_weights(&p2.roots, &gauge)), - [WeightDomain::NonNegative { - proof: NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples - }] - ); - assert_eq!( - domains(&with_confirmed_weights(&p2.roots, &data())), - [WeightDomain::UnknownOrSigned] - ); - // Nothing open: the same roots. - let exact = &candidates()[0]; - let QueryRoot::Operator(before) = &exact.roots[0] else { - panic!() - }; - let QueryRoot::Operator(after) = &with_confirmed_weights(&exact.roots, &counter)[0] else { - panic!() - }; - assert!(Rc::ptr_eq(before, after)); - } - - /// Each node is priced under its DAG id, and the total is the sum; - /// every candidate is either selected or rejected as costlier. The - /// CountSketch+heap estimate has no analytical guarantee, so deployment - /// evidence admits it. - #[test] - fn per_node_costs_cover_the_dag_and_sum_to_the_total() { - let candidates = candidates(); - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let selection = select_among_plans( - &candidates, - &[entry_with( - target - .clone() - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1), - repeating(10_000), - )], - &data(), - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredExact), - ) - .unwrap(); - for candidate in candidates.iter().filter(|c| c.id != "P2") { - let cost = &selection.costs[&candidate.id]; - let ids: Vec<_> = candidate.dag.nodes.iter().map(|n| n.id).collect(); - let mut keys: Vec<_> = cost.per_node.keys().copied().collect(); - keys.sort(); - let mut sorted = ids.clone(); - sorted.sort(); - assert_eq!(keys, sorted); - let sum: f64 = cost - .per_node - .values() - .map(|n| n.cost_per_second.unwrap()) - .sum(); - assert_eq!(cost.cost_per_second.unwrap(), sum); - assert!(cost.cost_per_second.unwrap() > 0.0); - } - let costlier: Vec<_> = selection - .rejected - .iter() - .filter(|r| r.cause == RejectionCause::Costlier) - .collect(); - assert_eq!(costlier.len(), 1); - assert_ne!(costlier[0].id, selection.selected); - } - - /// Query 1 is `sum(...)` over query 2's output. Every node reached by - /// both queries is `NotMaterialized`, so recomputed for each: charged - /// twice. Materializing query 2's root (`MatAtQuery`) charges it once, - /// and the nodes below it once, since they run only when it does. - #[test] - fn not_materialized_shared_nodes_are_charged_per_query() { - let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::exact()); - let inner = outer.children()[0].clone(); - let mut candidate = optimize_physical_asap( - "L", - &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], - &[], - &data(), - false, - ) - .unwrap() - .remove(0); - let [outer, inner] = candidate.dag.roots[..] else { - panic!("two roots") - }; - // Kept state is priced apart; this compares computations only. - let settings = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let cost = price(&candidate.dag, &recurrences(2), &data(), &settings).unwrap(); - let multipliers: BTreeMap<_, _> = cost - .per_node - .iter() - .map(|(id, c)| (*id, c.multiplier)) - .collect(); - assert_eq!(multipliers[&outer], 1); - assert_eq!(multipliers[&inner], 2); - assert!(multipliers.len() > 2, "nodes below the shared root"); - for (id, times) in &multipliers { - if *id != outer { - assert_eq!(*times, 2, "node {id} is read by both queries"); - } - } - - let node = candidate - .dag - .nodes - .iter_mut() - .find(|n| n.id == inner) - .unwrap(); - node.materialization = Materialization::MatAtQuery(asap_types::physical::Medium::Memory); - let stored = price(&candidate.dag, &recurrences(2), &data(), &settings).unwrap(); - for (id, c) in &stored.per_node { - assert_eq!(c.multiplier, 1, "node {id} is computed once"); - let once = cost.per_node[id].cost_per_second.unwrap() / multipliers[id] as f64; - assert!( - (c.cost_per_second.unwrap() - once).abs() <= 1e-12 * once, - "node {id}" - ); - } - assert!(stored.cost_per_second.unwrap() < cost.cost_per_second.unwrap()); - } - - /// A `NotMaterialized` node two queries read is computed for each, so - /// its working memory is held once per query, each time for that - /// query's latency: `bytes · Σ rate_q · latency_q`, not the longest - /// latency at both rates. Query 1 is `sum(...)` over query 2's output, - /// so it runs longer; they repeat every 10 s and every 20 s. - #[test] - fn shared_not_materialized_memory_is_held_for_each_reading_query() { - let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::exact()); - let inner = outer.children()[0].clone(); - let candidate = optimize_physical_asap( - "L", - &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], - &[], - &data(), - false, - ) - .unwrap() - .remove(0); - let recurrence = [repeating(10_000), repeating(20_000)]; - let cost = |price_per_byte_second| { - let settings = unit_costs(|c| c.cost_per_memory_byte_second = price_per_byte_second); - price(&candidate.dag, &recurrence, &data(), &settings).unwrap() - }; - let (with, without) = (cost(1.0), cost(0.0)); - let [outer_ms, inner_ms] = [0, 1].map(|q| with.usage.latency_per_query[q].total_ms()); - assert!(outer_ms > inner_ms, "{outer_ms} vs {inner_ms}"); - let mut checked = 0; - for (id, node) in &with.per_node { - let Some(held) = node.detail.split("; holds ").nth(1) else { - continue; - }; - if node.multiplier != 2 { - continue; - } - let bytes: f64 = held.split(' ').next().unwrap().parse().unwrap(); - let term = - node.cost_per_second.unwrap() - without.per_node[id].cost_per_second.unwrap(); - let expected = bytes * (0.1 * outer_ms + 0.05 * inner_ms) / 1_000.0; - assert!(close(term, expected), "node {id}: {term} vs {expected}"); - checked += 1; - } - assert!(checked > 0, "a shared node holds memory"); - } - - /// Every combination of Pass 1 candidates of one query, as Stage 2 - /// candidates `P1`, `P2`, ... - fn all_candidates(query: &str, target: &AccuracyTarget) -> Vec { - all_candidates_sized(query, target, &|roots| roots) - } - - /// [`all_candidates`] with each candidate's logical roots passed through - /// `size` before Stage 2. - fn all_candidates_sized( - query: &str, - target: &AccuracyTarget, - size: &dyn Fn(Vec) -> Vec, - ) -> Vec { - let root = lower_promql(query, target.clone()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let mut choices = vec![vec![]]; - for target in &inventory.targets { - choices = choices - .into_iter() - .flat_map(|choice: Vec| { - (0..target.choice_count()).map(move |index| { - let mut choice = choice.clone(); - choice.push(index); - choice - }) - }) - .collect(); - } - choices - .iter() - .enumerate() - .map(|(index, choice)| { - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, choice, - ) - .unwrap(); - let roots = size(roots); - let mut candidate = optimize_physical_asap("L", &roots, &[], &data(), false) - .unwrap() - .remove(0); - candidate.id = format!("P{}", index + 1); - candidate - }) - .collect() - } - - fn has_summary_estimate(candidate: &PhysicalWorkloadPlan) -> bool { - candidate - .dag - .nodes - .iter() - .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryEstimate { .. }))) - } - - /// [`per_series_kll`]'s candidates with their summaries sized as Pass 1 - /// sizes them for the target over `data` ([`sized_as_pass1`]). - fn per_series_kll_sized_as_pass1( - scope: AccuracyScope, - data: &DataWorkload, - ) -> (Vec, AccuracyTarget) { - let (_, target) = per_series_kll(scope); - let requested = target.clone().accuracy_scope(AccuracyScope::PerValue); - let candidates = - all_candidates_sized("quantile_over_time(0.99, x[5m])", &requested, &|roots| { - sized_as_pass1(&roots, std::slice::from_ref(&target), data) - }) - .into_iter() - .filter(|c| !kll_params(&c.roots).is_empty()) - .collect(); - (candidates, target) - } - - /// The KLL candidates of `quantile_over_time(0.99, x[5m])`, per series, - /// within rank error 0.01 at δ = 0.01 over `scope`. - fn per_series_kll(scope: AccuracyScope) -> (Vec, AccuracyTarget) { - let requested = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Rank); - let candidates = all_candidates("quantile_over_time(0.99, x[5m])", &requested) - .into_iter() - .filter(|c| !kll_params(&c.roots).is_empty()) - .collect::>(); - assert!(!candidates.is_empty()); - let target = requested - .clone() - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(scope); - (candidates, target) - } - - /// Bad input: a workload with a different number of entries than a - /// candidate has query roots is an error, not a rejection. - #[test] - fn an_entry_count_mismatch_is_an_error() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let entries = vec![entry_with(target, repeating(10_000)); 2]; - let Err(ErrorSelectingPlan::EntryCount { - candidate, - roots, - entries, - }) = select_among_plans(&candidates(), &entries, &data(), SelectionModels::builtin()) - else { - panic!("an entry count mismatch is an error") - }; - assert_eq!((candidate.as_str(), roots, entries), ("P1", 1, 2)); - } - - fn kll_params(roots: &[QueryRoot]) -> Vec { - roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::Kll => Some(kind.params().clone()), - _ => None, - }) - .collect() - } - - /// Stage 3 only checks: a KLL Pass 1 sized per value (δ) misses a - /// whole-result target over `G` = 10 000 series and is rejected, not - /// enlarged; sized as Pass 1 sizes it for that target (δ / G = 1e-6, a - /// larger `k`), it is valid, and the selected plan runs that size. - #[test] - fn a_whole_result_target_is_checked_not_resized() { - let (per_value_sized, whole) = per_series_kll(AccuracyScope::WholeResult); - let entries = [entry_with(whole.clone(), repeating(10_000))]; - let Err(ErrorSelectingPlan::NoValidCandidate(rejected)) = select_among_plans( - &per_value_sized, - &entries, - &data(), - SelectionModels::builtin(), - ) else { - panic!("a per-value size misses the whole-result target") - }; - assert!(rejected - .iter() - .all(|r| r.cause == RejectionCause::AccuracyTargetMissed)); - let (sized, _) = per_series_kll_sized_as_pass1(AccuracyScope::WholeResult, &data()); - let selection = - select_among_plans(&sized, &entries, &data(), SelectionModels::builtin()).unwrap(); - let ( - SketchParams::Kll { k: per_value_k, .. }, - SketchParams::Kll { - k, - failure_probability, - }, - ) = ( - &kll_params(&per_value_sized[0].roots)[0], - &kll_params(&selection.plan)[0], - ) - else { - panic!("KLL builds") - }; - assert_eq!(*failure_probability, Some(0.01 / 10_000.0)); - assert!(k > per_value_k, "{k} vs {per_value_k}"); - let selected = sized.iter().find(|c| c.id == selection.selected).unwrap(); - assert_eq!(kll_params(&selection.plan), kll_params(&selected.roots)); - } - - /// With its metric declared a counter, the whole-expression CMS+heap - /// candidate (P2) proves its weights non-negative, is valid and is - /// priced: its per-node costs cover its DAG and sum to its total. - #[test] - fn a_declared_counter_prices_the_count_min_heap_candidate() { - use asap_types::workload::PrometheusMetricType; - let candidates = candidates(); - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let counter = with_metric_types(data(), &[("m", PrometheusMetricType::Counter)]); - let selection = select_among_plans( - &candidates, - &[entry_with(target, repeating(10_000))], - &counter, - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredExact), - ) - .unwrap(); - let p2 = candidates.iter().find(|c| c.id == "P2").unwrap(); - let cost = &selection.costs["P2"]; - let mut ids: Vec<_> = p2.dag.nodes.iter().map(|n| n.id).collect(); - ids.sort(); - let mut keys: Vec<_> = cost.per_node.keys().copied().collect(); - keys.sort(); - assert_eq!(keys, ids); - let sum: f64 = cost - .per_node - .values() - .map(|n| n.cost_per_second.unwrap()) - .sum(); - assert_eq!(cost.cost_per_second.unwrap(), sum); - } - - /// The KLL failure probability each selected sketch is sized for. - fn kll_failure_probabilities(roots: &[QueryRoot]) -> Vec> { - kll_params(roots) - .into_iter() - .map(|params| match params { - SketchParams::Kll { - failure_probability, - .. - } => failure_probability, - other => panic!("{other:?}"), - }) - .collect() - } - - /// A target on the groups holding at least 5% of the records covers at - /// most 20 of them, so Pass 1 sizes each sketch for δ / 20 and the target is - /// met even when the number of values the result returns is unbound - /// (here a source with no declared series count), where a whole-result - /// target stays unmet. With 10 000 series declared it still needs only - /// δ / 20, not the whole result's δ / 10 000. - #[test] - fn a_record_share_target_sizes_each_sketch_for_one_over_share_groups() { - for data in [with_series(None), data()] { - let (candidates, five_percent) = per_series_kll_sized_as_pass1( - AccuracyScope::GroupsWithRecordShareAtLeast(0.05), - &data, - ); - let selection = select_among_plans( - &candidates, - &[entry_with(five_percent.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) - .expect("a KLL candidate meets the 5% target"); - assert_eq!( - kll_failure_probabilities(&selection.plan), - vec![Some(0.01 / 20.0)] - ); - } - } - - /// Without a bound on the values one evaluation returns, a per-value - /// guarantee does not meet a whole-result target: a PromQL vector whose - /// source declares no series count, and a SQL relation (until a bound on - /// its groups is an input). A scalar returns one value. - #[test] - fn an_unbound_number_of_values_leaves_a_whole_result_target_unmet() { - let (candidates, whole) = per_series_kll(AccuracyScope::WholeResult); - let no_series_count = with_series(None); - let Err(ErrorSelectingPlan::NoValidCandidate(rejected)) = select_among_plans( - &candidates, - &[entry_with(whole, repeating(10_000))], - &no_series_count, - SelectionModels::builtin(), - ) else { - panic!("the KLL candidates are invalid") - }; - assert!( - rejected[0].reason.contains("is not bound"), - "{}", - rejected[0].reason - ); - let table = asap_logical_asap_optimization::test_support::scan_from( - asap_types::ir::operator::Source::Table { - table_ref: "t".into(), - }, - Schema::new(vec![asap_types::ir::schema::Field::plain( - "v", - DataType::Float64, - false, - )]), - ); - assert_eq!(values_of(&QueryRoot::Operator(table), &data()), None); - assert_eq!( - values_of( - &QueryRoot::Scalar(ScalarExpr::literal_f64(1.0)), - &DataWorkload::default() - ), - Some(1) - ); - } - - /// A root that returns at most one row has `G` = 1, provable from the - /// query with no data workload: a global aggregate, PromQL or SQL, and - /// what selects, renames or joins single rows above it. A grouped SQL - /// aggregate stays unbound, and a grouped PromQL one, or a global top-k - /// (k rows), is bound by the series count of the source it reads. - #[test] - fn a_root_of_at_most_one_row_returns_one_value() { - let none = DataWorkload::default(); - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - assert_eq!(values_of(&root("sum(rate(x[5m]))"), &none), Some(1)); - assert_eq!(values_of(&root("topk(5, rate(x[5m]))"), &none), None); - assert_eq!(values_of(&root("sum by (job) (rate(x[5m]))"), &none), None); - assert_eq!( - values_of(&root("sum by (job) (rate(x[5m]))"), &data()), - Some(10_000) - ); - let table = || { - asap_logical_asap_optimization::test_support::scan_from( - asap_types::ir::operator::Source::Table { - table_ref: "t".into(), - }, - Schema::new(vec![ - asap_types::ir::schema::Field::plain("k", DataType::Utf8, false), - asap_types::ir::schema::Field::plain("v", DataType::Float64, false), - ]), - ) - }; - let count = || AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }; - let global = asap_logical_asap_optimization::test_support::agg(vec![], count(), table()); - let grouped = asap_logical_asap_optimization::test_support::agg(vec![0], count(), table()); - assert_eq!( - values_of(&QueryRoot::Operator(global.clone()), &none), - Some(1) - ); - assert_eq!(values_of(&QueryRoot::Operator(grouped), &data()), None); - let cross = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Join { - kind: asap_types::ir::operator::operator_properties::JoinKind::Cross, - pred: asap_types::ir::Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), - left: global.clone(), - right: global, - })) - .unwrap(); - assert_eq!(values_of(&QueryRoot::Operator(cross), &none), Some(1)); - } - - /// `G` and the input row count are bound per query from the sources - /// it reads: another source's series do not count, a query over two - /// sources is bound by their sum, and one source without a series - /// count leaves it unbound. Pricing reads the same per-source series, with - /// the default for an undeclared source. - #[test] - fn values_per_result_is_bound_by_the_series_of_the_sources_read() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let series = |declared: &[(&str, u64)]| DataWorkload { - sources: declared - .iter() - .map(|(source, series)| { - let facts = SourceFacts { - series_count: Some(*series), - ..SourceFacts::default() - }; - (source.to_string(), facts) - }) - .collect(), - ..DataWorkload::default() - }; - let x = root("sum by (job) (rate(x[5m]))"); - let both = root("sum by (job) (rate(x[5m])) + sum by (job) (rate(m[5m]))"); - assert_eq!(values_of(&x, &series(&[("x", 10)])), Some(10)); - assert_eq!(values_of(&x, &series(&[("m", 10)])), None); - let declared = series(&[("x", 10), ("m", 20)]); - assert_eq!(values_of(&x, &declared), Some(10)); - assert_eq!(values_of(&both, &declared), Some(30)); - assert_eq!(series_read(&both, &declared), Some(30)); - assert_eq!(values_of(&both, &series(&[("x", 10)])), None); - // Pricing sums the series of the sources a node reads; an - // undeclared source, or none, leaves them unknown. - assert_eq!(declared_series(&["x", "m"].into(), &declared), Some(30)); - assert_eq!(declared_series(&["x", "y"].into(), &declared), None); - assert_eq!(declared_series(&BTreeSet::new(), &declared), None); - } - - /// The exact physical DAG of `queries` planned together, over `data`. - fn exact_dag(queries: &[&str], data: &DataWorkload) -> PhysicalASAPDAG { - let roots: Vec<_> = queries - .iter() - .map(|query| { - let root = lower_promql(query, AccuracyTarget::exact()); - QueryRoot::Operator(root) - }) - .collect(); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let choice = vec![0; inventory.targets.len()]; - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - optimize_physical_asap("L1", &roots, &[], data, false) - .unwrap() - .remove(0) - .dag - } - - /// A node's operator, timing, work and price, to compare nodes across - /// DAGs whose ids differ. - fn node_prices(cost: &PlanCost) -> Vec<(OperatorKind, ExecutionTiming, String, f64)> { - cost.per_node - .values() - .map(|node| { - ( - node.operator, - node.timing, - node.detail.clone(), - node.cost_per_second.unwrap(), - ) - }) - .collect() - } - - /// No fact bounds the rows of a scan whose source declares neither an - /// arrival rate (`rows_per_second`, or `series_count` with - /// `sample_interval`) nor `rows_held`: pricing refuses, naming the - /// source and the missing fact, instead of assuming a rate. - #[test] - fn a_scan_without_a_row_fact_is_refused_naming_its_source() { - let data = DataWorkload::default(); - let error = price( - &exact_dag(&["sum(rate(m[5m]))"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap_err() - .1; - assert!( - matches!(&error, AnalyticalCostError::MissingSourceFact { source_name, .. } if source_name == "m"), - "{error}" - ); - assert!(error.to_string().contains("rows_held"), "{error}"); - } - - // #955 example 1: `rate(m[5m] offset 1h)` at query time reads the 5 - // minutes `(T - 65m, T - 60m]`, so it scans the samples `rate(m[5m])` - // scans but keeps and prices the 65 minutes of raw `m` `rate(m[65m])` - // keeps. - #[test] - fn an_offset_window_at_query_time_keeps_its_whole_raw_lookback() { - // The scan's samples read and raw data bytes kept, from its detail. - let scan_of = |query: &str| { - let dag = exact_dag(&[query], &data()); - let scan = node_of(&dag, is_scan); - let cost = price(&dag, &recurrences(1), &data(), &CostInputs::default()).unwrap(); - let detail = &cost.per_node[&scan].detail; - let kept_bytes: u64 = detail - .split("; keeps ") - .find_map(|part| { - part.split(" bytes of raw data on disk") - .next()? - .parse() - .ok() - }) - .unwrap_or_else(|| panic!("no raw data kept: {detail}")); - (scanned_samples(&cost, scan), kept_bytes) - }; - let (offset_samples, offset_kept) = scan_of("sum(rate(m[5m] offset 1h))"); - let (five_minutes_samples, five_minutes_kept) = scan_of("sum(rate(m[5m]))"); - let (_, sixty_five_minutes_kept) = scan_of("sum(rate(m[65m]))"); - assert_eq!(offset_samples, five_minutes_samples); - assert_eq!(offset_kept, sixty_five_minutes_kept); - assert_eq!(offset_kept, 13 * five_minutes_kept); - } - - /// Data at rest is bounded by the rows its source holds: a 5-minute - /// window of `m`, which holds 5,000 rows and declares no arrival rate, - /// reads at most those 5,000. - #[test] - fn rows_held_bound_a_scan_of_data_at_rest() { - let data = DataWorkload { - arrival: DataArrival::AtRest, - sources: [( - "m".to_string(), - SourceFacts { - rows_held: Some(5_000), - ..SourceFacts::default() - }, - )] - .into(), - }; - let cost = price( - &exact_dag(&["sum(rate(m[5m]))"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - let details: Vec = node_prices(&cost) - .into_iter() - .map(|(_, _, d, _)| d) - .collect(); - assert!( - details.iter().any(|d| d.starts_with("scan 5000 samples;")), - "{details:?}" - ); - } - - /// With no declared `series_count`, a per-series reduction is priced at - /// one state per row read (each series has at least one), not at an - /// assumed series count: 1,000 rows per second over 5 minutes. - #[test] - fn undeclared_series_are_bound_by_the_rows_read() { - let data = DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000.0, - }, - sources: [("m".to_string(), SourceFacts::default())].into(), - }; - let cost = price( - &exact_dag(&["sum(rate(m[5m]))"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - let details: Vec = node_prices(&cost) - .into_iter() - .map(|(_, _, d, _)| d) - .collect(); - assert!( - details.iter().any(|d| d.contains("into 300000 states")), - "{details:?}" - ); - } - - /// Each scan is priced at its own source's rate, `series_count` series - /// each sampled every `sample_interval`, not at the whole workload's - /// `rows_per_second`: adding a query on a second metric, which a - /// runtime declares by raising `rows_per_second` by that metric's rate, - /// leaves the first metric's node prices unchanged. - #[test] - fn a_query_on_another_metric_leaves_a_metrics_node_prices_unchanged() { - // `a` and `b`: 10 000 series sampled every 100 ms, 100 000 rows/s each. - let ingesting = |rows_per_second| DataWorkload { - arrival: DataArrival::Ingesting { rows_per_second }, - sources: ["a", "b"] - .into_iter() - .map(|metric| { - let facts = SourceFacts { - series_count: Some(10_000), - sample_interval: Some(asap_types::workload::DurationMs(100)), - ..SourceFacts::default() - }; - (metric.to_string(), facts) - }) - .collect(), - }; - let settings = CostInputs::default(); - let on_a = "sum(sum_over_time(a[1m]))"; - let alone = ingesting(100_000.0); - let alone = price( - &exact_dag(&[on_a], &alone), - &recurrences(1), - &alone, - &settings, - ) - .unwrap(); - let joint = ingesting(200_000.0); - let joint = price( - &exact_dag(&[on_a, "sum(sum_over_time(b[1m]))"], &joint), - &recurrences(2), - &joint, - &settings, - ) - .unwrap(); - let mut joint_nodes = node_prices(&joint); - for node in node_prices(&alone) { - let found = joint_nodes.iter().position(|other| *other == node); - assert!(found.is_some(), "{node:?} not in {joint_nodes:#?}"); - joint_nodes.remove(found.unwrap()); - } - // Each metric's scan reads its own 100 000 rows per second. - assert_eq!( - joint.usage.scanned_bytes_per_second, - 2.0 * alone.usage.scanned_bytes_per_second - ); - } - - /// A partitioned sort and limit (`topk by (job)`) take their partition - /// count from `job`'s declared distinct values. With `job` undeclared, - /// no count is invented: each series is in one partition and each row - /// read in at most one, so the partitions are at most the series and - /// at most the rows the sort reads. - #[test] - fn a_partitioned_topk_has_one_partition_per_declared_label_value() { - let details = |declared: &[(&str, u64)], series: u64| { - let mut data = with_series(Some(series)); - data.sources.get_mut("m").unwrap().distinct_values_per_label = declared - .iter() - .map(|(label, count)| (label.to_string(), *count)) - .collect(); - let cost = price( - &exact_dag(&["topk by (job) (3, m)"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - node_prices(&cost) - .into_iter() - .filter(|(operator, ..)| { - matches!(operator, OperatorKind::Sort | OperatorKind::Limit) - }) - .map(|(_, _, detail, _)| detail) - .collect::>() - }; - let declared = details(&[("job", 10)], 10_000); - assert!( - declared.iter().any(|d| d.contains(" in 10 partitions;")), - "{declared:?}" - ); - assert!( - declared.iter().any(|d| d.starts_with("limit to 30 rows;")), - "{declared:?}" - ); - // Undeclared: the 2,000 rows read, fewer than the 10,000 series. - let undeclared = details(&[("instance", 10)], 10_000); - assert!( - undeclared - .iter() - .any(|d| d.starts_with("sort 2000 rows in 2000 partitions;")), - "{undeclared:?}" - ); - assert!( - undeclared - .iter() - .any(|d| d.starts_with("limit to 2000 rows;")), - "{undeclared:?}" - ); - // 50 series hold at most 50 groups, declared or not. - let few = details(&[("job", 10_000)], 50); - assert!( - few.iter().any(|d| d.contains(" in 50 partitions;")), - "{few:?}" - ); - let few = details(&[], 50); - assert!( - few.iter().any(|d| d.contains(" in 50 partitions;")), - "{few:?}" - ); - } - - /// `x` holding 10,000 series over 4 regions, 25 services and 625 - /// endpoints, each endpoint in one service (the eval's RollupHeavy - /// `http`), with `(service, endpoint)`'s 625 combinations declared when - /// `declared_set`. - fn hierarchical_labels(declared_set: bool) -> DataWorkload { - let mut data = with_series(Some(10_000)); - let facts = data.sources.get_mut("x").unwrap(); - facts.distinct_values_per_label = [ - ("region".to_string(), 4), - ("service".to_string(), 25), - ("endpoint".to_string(), 625), - ] - .into(); - if declared_set { - facts.distinct_values_per_label_set = vec![LabelSetDistinctValues { - labels: vec!["service".to_string(), "endpoint".to_string()], - distinct_values: 625, - }]; - } - data - } - - /// A grouping is priced at the smallest bound the declared facts give: - /// the declared `(service, endpoint)` set's 625 combinations, not - /// 25 x 625 capped at the 10,000 series; a superset grouping at the set - /// times `region`'s 4; without the set, at the per-label product capped - /// at the series. A partitioned top-k takes the same count. - #[test] - fn a_grouping_is_priced_at_its_declared_label_set_count() { - let details = |query: &str, declared_set: bool| { - let data = hierarchical_labels(declared_set); - let cost = price( - &exact_dag(&[query], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - node_prices(&cost) - .into_iter() - .map(|(_, _, detail, _)| detail) - .collect::>() - }; - let sum = |keys: &str| format!("sum by ({keys}) (sum_over_time(x[15m]))"); - for (query, declared_set, expected) in [ - (sum("service, endpoint"), true, "into 625 states"), - (sum("region, service, endpoint"), true, "into 2500 states"), - ( - sum("service, endpoint"), - false, - "Sum accumulator into 10000 states", - ), - (sum("region, service"), true, "into 100 states"), - ( - "topk by (service, endpoint) (3, sum_over_time(x[15m]))".to_string(), - true, - " in 625 partitions;", - ), - // A relabeled key no longer holds the declared set's values: - // `endpoint` now joins `region` and `endpoint` (up to 4 x 625). - ( - r#"sum by (service, endpoint) (label_join(sum_over_time(x[15m]), "endpoint", "-", "region", "endpoint"))"#.to_string(), - true, - "into 10000 states", - ), - // A binary operator pairs the labels of two rows: per-label only. - ( - "sum by (service, endpoint) (sum_over_time(x[15m]) / on (service) group_left (endpoint) sum_over_time(x[15m]))".to_string(), - true, - "into 10000 states", - ), - ] { - let details = details(&query, declared_set); - assert!( - details.iter().any(|d| d.contains(expected)), - "{query} (set declared: {declared_set}): {details:#?}" - ); - } - } - - /// A grouped root's `G` takes the declared label set's count too: the - /// set's 625 for `(service, endpoint)`, times `region`'s 4 for a - /// superset; without the set, the per-label product capped at the - /// series. - #[test] - fn values_per_result_is_bound_by_a_declared_label_set() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - for (query, declared_set, values) in [ - ("sum by (service, endpoint) (rate(x[5m]))", true, 625), - ( - "sum by (region, service, endpoint) (rate(x[5m]))", - true, - 2_500, - ), - ("sum by (service, endpoint) (rate(x[5m]))", false, 10_000), - ] { - assert_eq!( - values_of(&root(query), &hierarchical_labels(declared_set)), - Some(values), - "{query} (set declared: {declared_set})" - ); - } - } - - /// A grouped root's `G` is also bound by its keys' declared distinct - /// values, at most the series read: `by (job)` with 10 jobs returns at - /// most 10 values, also below a sort; a key without a declared count, a - /// relabeled key, `without (...)` and a top-k (several rows per group) - /// keep the series bound. - #[test] - fn values_per_result_is_bound_by_the_declared_distinct_values_of_its_keys() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let mut data = with_series(Some(10_000)); - data.sources.get_mut("x").unwrap().distinct_values_per_label = - [("job".to_string(), 10), ("instance".to_string(), 5_000)].into(); - for (query, values) in [ - ("sum by (job) (rate(x[5m]))", 10), - ("sum by (job, instance) (rate(x[5m]))", 10_000), - ("sum by (job, region) (rate(x[5m]))", 10_000), - ("sum without (instance) (rate(x[5m]))", 10_000), - ("topk by (job) (3, rate(x[5m]))", 10_000), - ("sum by (job) (rate(x[5m]) + rate(m[5m]))", 20_000), - // A relabeled key no longer holds the declared label's values. - ( - r#"sum by (job) (label_replace(rate(x[5m]), "job", "$1", "instance", "(.*)"))"#, - 10_000, - ), - ("sort_desc(sum by (job) (rate(x[5m])))", 10), - // `count_values("job", ...)` overwrites `job` with sample values. - ( - r#"sum by (job) (count_values by (job) ("job", rate(x[5m])))"#, - 10_000, - ), - ] { - assert_eq!(values_of(&root(query), &data), Some(values), "{query}"); - } - } - - /// `G` counts the values the root can return: an `or` returns the - /// values of both branches (here up to `series(x)` series plus the - /// groups of `x`), a constant vector one more, while a join or a fold - /// of one source counts it once. - #[test] - fn values_per_result_adds_the_branches_of_an_or() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let data = with_series(Some(10)); - for (query, values) in [ - ("rate(x[5m]) or sum without (instance) (rate(x[5m]))", 20), - ("rate(x[5m]) or vector(0)", 11), - ("rate(x[5m]) or rate(m[5m])", 20), - ("(rate(x[5m]) or vector(0)) or vector(1)", 12), - ("sum by (job) (rate(x[5m]) or rate(x[5m]))", 20), - ("rate(x[5m]) / rate(x[5m])", 10), - ("sum without (instance) (rate(x[5m])) * rate(x[5m])", 10), - ("rate(x[5m]) + rate(m[5m])", 20), - ] { - assert_eq!(values_of(&root(query), &data), Some(values), "{query}"); - } - } - - /// `vector(s)` returns one row, but the operators below it still read - /// their sources' rows: the input row count of - /// `vector(scalar(max(rate(x[5m]))))` counts the 10 series the `max` - /// folds (plus the one constant row), never shrinking going up the tree, - /// while its `G` is 1. - #[test] - fn a_vector_of_a_scalar_keeps_the_rows_read_below_it() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let data = with_series(Some(10)); - let vector = root("vector(scalar(max(rate(x[5m]))))"); - assert_eq!(series_read(&vector, &data), Some(11)); - assert_eq!(values_of(&vector, &data), Some(1)); - assert_eq!(values_of(&root("vector(1)"), &data), Some(1)); - let or = root("rate(x[5m]) or vector(scalar(max(rate(x[5m]))))"); - assert_eq!(series_read(&or, &data), Some(21)); - } - - /// Deployment evidence that every output was measured exact. - struct EveryOutputMeasuredExact; - - impl AccuracyEvidenceProvider for EveryOutputMeasuredExact { - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - Some(AccuracyGuarantee::exact("measured")) - } - } - - /// Deployment evidence of a small error in a fixed metric for every - /// output that is not exact. - struct EveryOutputMeasuredIn(AccuracyMetric); - - impl AccuracyEvidenceProvider for EveryOutputMeasuredIn { - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - Some(AccuracyGuarantee::new( - self.0, - BoundExpr::Constant { value: 1e-6 }, - ProbabilityExpr::Constant { value: 1e-6 }, - vec![], - )) - } - } - - /// A bound stated in another metric misses the target however small it - /// is: a relative bound does not bound a top-k frequency's L1-scaled - /// error, so only the exact candidate stays valid; the same evidence in - /// the target's metric admits the sketches. - #[test] - fn a_bound_in_another_metric_misses_the_target() { - let counter = with_metric_types(data(), &[("m", PrometheusMetricType::Counter)]); - let target = AccuracyTarget::max_error(0.5) - .failure_probability(0.5) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let select = |metric| { - select_among_plans( - &candidates(), - &[entry_with(target.clone(), repeating(10_000))], - &counter, - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredIn(metric)), - ) - .unwrap() - }; - let relative = select(AccuracyMetric::Relative); - assert_eq!(relative.selected, "P1"); - let mut invalid: Vec<_> = relative - .rejected - .iter() - .filter(|r| r.cause.failed_a_check()) - .collect(); - invalid.sort_by(|a, b| a.id.cmp(&b.id)); - assert_eq!( - invalid.iter().map(|r| r.id.as_str()).collect::>(), - ["P2", "P3"] - ); - for rejected in invalid { - assert!( - rejected.reason.contains("no evidence that it meets"), - "{}", - rejected.reason - ); - } - let l1 = select(AccuracyMetric::ABSOLUTE_STREAM_L1); - assert!( - l1.rejected.iter().all(|r| !r.cause.failed_a_check()), - "{:?}", - l1.rejected - ); - } - - /// A difference of two quantile estimates has no output guarantee, though - /// each estimate is certified: its summary candidates stay listed as - /// invalid with a reason. The deployment's empirical accuracy for the - /// output admits them. - #[test] - fn combined_estimates_need_evidence_about_the_output() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates( - "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])", - &target, - ); - let summaries: Vec<_> = candidates - .iter() - .filter(|c| has_summary_estimate(c)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!summaries.is_empty()); - - let builtin = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), - repeating(10_000), - )], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - for id in &summaries { - let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, RejectionCause::NoAccuracyGuarantee); - assert!( - rejection.reason.contains("no guarantee"), - "{}", - rejection.reason - ); - } - assert!(!summaries.contains(&builtin.selected.as_str())); - - let measured = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), - repeating(10_000), - )], - &data(), - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredExact), - ) - .unwrap(); - for id in &summaries { - assert!(measured.costs.contains_key(*id), "{id} is valid"); - } - } - - /// Deployment evidence of a large absolute error (1.0) for every output. - struct EveryOutputMeasuredLoose; - - impl AccuracyEvidenceProvider for EveryOutputMeasuredLoose { - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - Some(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: 1.0 }, - ProbabilityExpr::Constant { value: 1e-6 }, - vec![], - )) - } - } - - /// A model that accepts every output directly, whatever its guarantee. - struct AcceptsEveryOutput; - - impl AccuracyModel for AcceptsEveryOutput { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - BuiltinAccuracyModel.satisfies(guarantee, target) - } - - fn accepts_output_regardless_of_guarantee( - &self, - _output: &QueryRoot, - _target: &AccuracyTarget, - ) -> bool { - true - } - } - - /// The hook overrides guarantees, not the executor's requirement on - /// Count-Min update weights: over samples of a metric declared a gauge, - /// the CmsWithHeap candidate (P2) is rejected as - /// `CountMinWeightsNotProvenNonNegative` under a model that accepts - /// every output, as under the built-in one. - #[test] - fn unproven_count_min_weights_reject_even_an_accepting_model() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - for accuracy in [ - &AcceptsEveryOutput as &dyn AccuracyModel, - &BuiltinAccuracyModel, - ] { - let selection = select_among_plans( - &candidates(), - &[entry_with(target.clone(), repeating(10_000))], - &gauge, - SelectionModels::builtin().with_accuracy(accuracy), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!( - p2.cause, - RejectionCause::CountMinWeightsNotProvenNonNegative, - "{}", - p2.reason - ); - assert!( - p2.reason.contains("not declared a counter"), - "{}", - p2.reason - ); - assert!(p2.cause.failed_a_check()); - } - } - - /// `IgnoreAccuracyGuarantees` (issue #774) does not override the - /// executor's requirement on Count-Min update weights either: over - /// samples of a gauge, P2's CmsWithHeap is still rejected as - /// `CountMinWeightsNotProvenNonNegative`. - #[test] - fn unproven_count_min_weights_reject_even_when_ignoring_guarantees() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - let settings = CostInputs { - ..CostInputs::default() - }; - let selection = select_among_plans( - &candidates(), - &[entry_with(target, repeating(10_000))], - &gauge, - SelectionModels::builtin() - .with_cost_inputs(&settings) - .with_accuracy(&asap_accuracy::IgnoreAccuracyGuarantees), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!( - p2.cause, - RejectionCause::CountMinWeightsNotProvenNonNegative, - "{}", - p2.reason - ); - } - - /// With `IgnoreAccuracyGuarantees` (issue #774), each summary - /// candidate the built-in model rejects (no guarantee, or one that - /// misses the target) is valid, and its accepted output records the - /// cause the guarantee check would have rejected it with; an output - /// whose guarantee meets the target records none. - #[test] - fn ignoring_guarantees_accepts_and_marks_outputs_the_check_rejects() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE); - let entries = [entry_with(target, repeating(10_000))]; - let settings = CostInputs { - ..CostInputs::default() - }; - let builtin = - select_among_plans(&candidates(), &entries, &data(), SelectionModels::builtin()) - .unwrap(); - let ignoring = select_among_plans( - &candidates(), - &entries, - &data(), - SelectionModels::builtin() - .with_cost_inputs(&settings) - .with_accuracy(&asap_accuracy::IgnoreAccuracyGuarantees), - ) - .unwrap(); - let mut marked = 0; - for rejection in builtin.rejected.iter().filter(|r| { - matches!( - r.cause, - RejectionCause::NoAccuracyGuarantee | RejectionCause::AccuracyTargetMissed - ) - }) { - let [accepted] = ignoring.accepted[&rejection.id].as_slice() else { - panic!("{} has one query", rejection.id) - }; - assert!(accepted.accepted_regardless_of_guarantee); - assert_eq!(accepted.guarantee_check_rejection, Some(rejection.cause)); - assert!(accepted.to_string().contains("ignoring"), "{accepted}"); - marked += 1; - } - assert!(marked > 0, "{:?}", builtin.rejected); - for (id, queries) in &builtin.accepted { - assert_eq!( - ignoring.accepted[id][0].guarantee_check_rejection, None, - "{id}" - ); - assert_eq!(queries[0].guarantee_check_rejection, None, "{id}"); - } - } - - /// `node` with every CmsWithHeap summary build replaced by a Hydra - /// Count-Min over the same update. - fn with_hydra_count_min(node: &Rc) -> Rc { - use asap_types::ir::schema::{HydraKind, HydraParams, SketchParams}; - let mut rebuilt = node.with_new_children(with_hydra_count_min).unwrap(); - if let Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) = &mut rebuilt.operator { - if let FieldDataType::Sketch(kind, _) = family { - if let SketchParams::CmsWithHeap { width, depth, .. } = kind.params() { - let (width, depth) = (*width, *depth); - *family = FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Cms, SketchParams::Cms { width, depth }), - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - }, - ); - } - } - } - Rc::new(rebuilt) - } - - /// A Hydra Count-Min over unproven weights is rejected by its own name: - /// the reason says `HydraCms`, not `Cms`. - #[test] - fn unproven_hydra_count_min_weights_name_hydra_cms() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - let mut p2 = candidates().into_iter().find(|c| c.id == "P2").unwrap(); - p2.roots = p2 - .roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryRoot::Operator(with_hydra_count_min(node)), - other => other.clone(), - }) - .collect(); - let CheckFailure { cause, reason, .. } = - accuracy_check(&p2, &[target], &gauge, &SelectionModels::builtin()).unwrap_err(); - assert_eq!(cause, RejectionCause::CountMinWeightsNotProvenNonNegative); - assert!( - reason.contains("HydraCms needs non-negative update weights"), - "{reason}" - ); - assert!(reason.contains("not declared a counter"), "{reason}"); - } - - /// The model's hook has priority over every guarantee. The summary - /// candidates of a difference of quantile estimates have no theoretical - /// guarantee: with evidence that misses the target the built-in model - /// rejects them as `AccuracyTargetMissed`, and with no evidence as - /// `NoAccuracyGuarantee`; a model that accepts every output accepts them - /// in both cases, regardless of guarantee. - #[test] - fn the_accuracy_model_accepts_an_output_whatever_its_guarantee() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates( - "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])", - &target, - ); - let summaries: Vec<_> = candidates - .iter() - .filter(|c| has_summary_estimate(c)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!summaries.is_empty()); - let select = |accuracy: &dyn AccuracyModel, evidence: &dyn AccuracyEvidenceProvider| { - select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), - repeating(10_000), - )], - &data(), - SelectionModels::builtin() - .with_accuracy(accuracy) - .with_evidence(evidence), - ) - .unwrap() - }; - for (evidence, builtin_cause) in [ - ( - &EveryOutputMeasuredLoose as &dyn AccuracyEvidenceProvider, - RejectionCause::AccuracyTargetMissed, - ), - (&NoAccuracyEvidence, RejectionCause::NoAccuracyGuarantee), - ] { - let builtin = select(&BuiltinAccuracyModel, evidence); - let accepting = select(&AcceptsEveryOutput, evidence); - for id in &summaries { - let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, builtin_cause, "{}", rejection.reason); - assert!(accepting.costs.contains_key(*id), "{id} is valid"); - assert!( - accepting.accepted[*id][0].accepted_regardless_of_guarantee, - "{id}" - ); - } - } - } - - /// `candidates` with every state node (a summary build or an exact - /// aggregate) kept at ingestion time, so selection charges its kept - /// bytes every second (a `MatAtQuery` state is held only for one - /// evaluation). - fn keeping_state(mut candidates: Vec) -> Vec { - for candidate in &mut candidates { - for node in &mut candidate.dag.nodes { - if matches!( - node.payload, - Payload::ASAP(ASAPOp::SummaryAgg { .. }) - | Payload::NonASAP(NonASAPOp::Aggregate { .. }) - ) { - node.materialization = - Materialization::MatAtIngestion(asap_types::physical::Medium::Memory); - } - } - } - candidates - } - - /// Kept exact quantile state keeps every value of its group: over one - /// group of 10 000 values it costs more than a quantile sketch sized for - /// ε = 0.01, δ = 0.01, which is selected. Over 100 groups of 100 values - /// (100 jobs declared), fewer than the sketch's `k`, the sketch keeps - /// every value too, in an item array grown past them, so its state is - /// no smaller than the exact state, which is selected. - #[test] - fn exact_quantile_state_costs_more_than_a_sketch_over_large_groups() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let mut data = data(); - data.sources.get_mut("x").unwrap().distinct_values_per_label = - [("job".to_string(), 100)].into(); - for (query, large_groups) in [ - ("quantile(0.99, rate(x[5m]))", true), - ("quantile by (job) (0.99, rate(x[5m]))", false), - ] { - let checked = target.clone().accuracy_metric(AccuracyMetric::Rank); - let candidates = keeping_state(all_candidates_sized(query, &target, &|roots| { - sized_as_pass1(&roots, std::slice::from_ref(&checked), &data) - })); - let selection = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::Rank), - repeating(10_000), - )], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let kept_bytes = |sketch: bool| { - let candidate = candidates - .iter() - .find(|c| { - has_summary_estimate(c) == sketch && selection.costs.contains_key(&c.id) - }) - .unwrap(); - selection.costs[&candidate.id].usage.average_memory_bytes - }; - let selected = candidates - .iter() - .find(|c| c.id == selection.selected) - .unwrap(); - match large_groups { - true => { - assert!(has_summary_estimate(selected), "{query}: {selection:?}"); - assert!(kept_bytes(true) < kept_bytes(false), "{query}"); - } - false => { - assert!(!has_summary_estimate(selected), "{query}: {selection:?}"); - assert!(kept_bytes(true) >= kept_bytes(false), "{query}"); - } - } - } - } - - /// Exact L2 state keeps one counter per distinct value of its group, so - /// it is priced like exact quantile state, which keeps every value. - #[test] - fn exact_l2_state_is_priced_like_exact_quantile_state() { - let aggregate_cost = |query: &str| { - let candidates = all_candidates(query, &AccuracyTarget::exact()); - let selection = select_among_plans( - &candidates, - &[entry(repeating(10_000))], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - let exact = candidates - .iter() - .find(|c| !has_summary_estimate(c)) - .unwrap(); - selection.costs[&exact.id] - .per_node - .values() - .find(|n| n.detail.starts_with("hash aggregate")) - .unwrap() - .cost_per_second - }; - assert_eq!( - aggregate_cost("l2_over_time(x[5m])"), - aggregate_cost("quantile_over_time(0.99, x[5m])") - ); - } - - /// Kept exact distinct-count state keeps every item of its group: over - /// 10 000 groups of 300 rows it is smaller than a distinct-count sketch - /// sized for ε = 0.01 and is selected; over one group of 3 000 000 rows a - /// sketch is. - #[test] - fn exact_distinct_count_beats_a_sketch_over_small_groups() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let one_series = with_series(Some(1)); - for (data, sketch_wins) in [(data(), false), (one_series, true)] { - let candidates = keeping_state(all_candidates("distinct_over_time(x[5m])", &target)); - let selection = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::Relative), - repeating(10_000), - )], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let selected = candidates - .iter() - .find(|c| c.id == selection.selected) - .unwrap(); - assert_eq!(has_summary_estimate(selected), sketch_wins, "{selection:?}"); - } - } - - /// The candidates of `query` that build a DDSketch. - fn ddsketch_candidates(candidates: &[PhysicalWorkloadPlan]) -> Vec { - candidates - .iter() - .filter(|c| { - c.dag.nodes.iter().any(|node| { - matches!(&node.payload, - Payload::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::DDSketch) - }) - }) - .map(|c| c.id.clone()) - .collect() - } - - /// A per-series DDSketch quantile is within a relative error, which - /// carries through `avg` and `sum` only over non-negative values, and - /// through `max` whatever their sign. With `x` declared a gauge whose - /// samples are non-negative, the DDSketch candidate of - /// `avg by (region) (quantile_over_time(0.99, x[100s]))` (and of its `sum`) - /// is valid and accepted with a relative-error guarantee; without the - /// declaration it is rejected. Even `max` needs the primitive interpolated - /// quantile contract, which can fail across zero. - #[test] - fn a_non_negative_gauge_carries_a_ddsketch_relative_bound_through_avg() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Relative); - // An `offset` moves the window, not the samples' sign. - for (outer, window, needs_sign) in [ - ("avg", "x[100s]", true), - ("sum", "x[100s]", true), - ("max", "x[100s]", true), - ("avg", "x[100s] offset 1h", true), - ] { - let query = format!("{outer} by (region) (quantile_over_time(0.99, {window}))"); - let candidates = all_candidates(&query, &target); - let ddsketch = ddsketch_candidates(&candidates); - assert!(!ddsketch.is_empty(), "{query}"); - for values_non_negative in [true, false] { - let mut data = with_metric_types(data(), &[("x", PrometheusMetricType::Gauge)]); - data.sources.get_mut("x").unwrap().values_non_negative = values_non_negative; - let selection = select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - for id in &ddsketch { - let accepted = selection.accepted.get(id); - if values_non_negative || !needs_sign { - let [accepted] = accepted.map(Vec::as_slice).unwrap_or_default() else { - panic!("{query}, {id} not valid: {:?}", selection.rejected); - }; - assert_eq!(accepted.accepted_metric, Some(AccuracyMetric::Relative)); - assert!(!accepted.accepted_regardless_of_guarantee); - } else { - assert!(accepted.is_none(), "{query}, {id}: {accepted:?}"); - let rejection = selection.rejected.iter().find(|r| &r.id == id).unwrap(); - assert!(rejection.cause.failed_a_check(), "{rejection:?}"); - } - } - } - } - } - - /// PromQL answers a quantile with φ outside [0, 1] as -Inf or +Inf, - /// which no sketch estimates and the executor's sketches refuse: such a - /// quantile gets no sketch candidate, so the exact plan is selected, and - /// a sketch never inherits its samples' sign for it, even over a gauge - /// declared non-negative. - #[test] - fn a_quantile_outside_the_unit_interval_is_answered_exactly() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Relative); - for query in [ - "avg by (region) (quantile_over_time(-1, x[100s]))", - "avg by (region) (quantile_over_time(2, x[100s]))", - "quantile_over_time(2, x[100s])", - "max by (region) (quantile_over_time(-1, x[100s]))", - ] { - let candidates = all_candidates(query, &target); - assert!( - candidates.iter().all(|c| !has_summary_estimate(c)), - "{query}: a sketch candidate is offered" - ); - let mut data = with_metric_types(data(), &[("x", PrometheusMetricType::Gauge)]); - data.sources.get_mut("x").unwrap().values_non_negative = true; - let selection = select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let selected = candidates - .iter() - .find(|c| c.id == selection.selected) - .unwrap(); - assert!( - !has_summary_estimate(selected), - "{query}: {}", - selection.selected - ); - } - } - - /// `topk(k, irate(m[w]))`, now a heavy-hitter ranking as `rate`'s is, - /// still plans: Stages 1-3 compose its candidates and select one. - #[test] - fn a_direct_irate_ranking_plans() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - for function in ["rate", "irate"] { - let query = format!("topk(10, {function}(data_total[5m]))"); - let candidates = all_candidates(&query, &target); - let data = with_metric_types(data(), &[("data_total", PrometheusMetricType::Counter)]); - let selection = select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ); - assert!(selection.is_ok(), "{query}: {selection:?}"); - } - } - - /// A candidate that ties the selected one is rejected for losing on - /// enumeration order, not as costlier. - #[test] - fn a_tie_is_reported_as_lost_on_enumeration_order() { - let mut candidates = candidates(); - candidates.truncate(1); - let mut twin = candidates[0].clone(); - twin.id = "P1-twin".into(); - candidates.push(twin); - let selection = select_among_plans( - &candidates, - &[entry(repeating(10_000))], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - assert_eq!(selection.selected, "P1"); - let reason = &selection.rejected[0].reason; - assert!( - reason.contains("tied with the selected candidate P1") - && reason.contains("enumeration order"), - "{reason}" - ); - } - - /// At δ = 0.001 a KLL candidate meets the target and is valid. - #[test] - fn kll_candidate_meets_a_failure_probability_below_one_percent() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates("quantile(0.99, rate(x[5m]))", &target); - let selection = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::Rank), - repeating(10_000), - )], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - let kll = candidates - .iter() - .find(|c| { - c.dag.nodes.iter().any(|n| { - matches!(&n.payload, Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if *kind.algorithm() == SketchAlgorithm::Kll) - }) - }) - .unwrap(); - assert!(selection.costs.contains_key(&kll.id), "{selection:?}"); - } - - /// `scalar(v)` inherits `v`'s guarantee; arithmetic over it composes - /// (`c · x` scales an absolute bound, `x − y` adds them); a rank error - /// does not compose (a rank interval counts as none); one over exact - /// operands only is exact. - #[test] - fn scalar_root_guarantees_compose_through_arithmetic() { - let node = lower_promql("sum(m)", AccuracyTarget::exact()); - let approximate = AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: 0.01 }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - let with = |guarantee: Option| { - ScalarExpr::PromqlScalarFromVector(Rc::new((*node).clone().with_guarantee(guarantee))) - }; - let doubled = |operand: ScalarExpr| { - QueryRoot::Scalar(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(operand), - right: Box::new(ScalarExpr::literal_f64(2.0)), - semantics: ExprSemantics::Promql, - }) - }; - assert_eq!( - root_guarantee(&QueryRoot::Scalar(with(Some(approximate.clone())))), - Some(approximate.clone()) - ); - let scaled = root_guarantee(&doubled(with(Some(approximate.clone())))).unwrap(); - assert_eq!(scaled.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert!((scaled.spec.error.evaluate().unwrap() - 0.02).abs() < 1e-12); - assert_eq!(scaled.spec.failure_probability.evaluate(), Some(0.001)); - let difference = QueryRoot::Scalar(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Sub, - left: Box::new(with(Some(approximate.clone()))), - right: Box::new(with(Some(approximate.clone()))), - semantics: ExprSemantics::Promql, - }); - let sum = root_guarantee(&difference).unwrap(); - assert!((sum.spec.error.evaluate().unwrap() - 0.02).abs() < 1e-12); - let mut rank = approximate; - rank.spec.accuracy_metric = AccuracyMetric::Rank; - assert_eq!(root_guarantee(&doubled(with(Some(rank)))), None); - assert!( - root_guarantee(&doubled(with(Some(AccuracyGuarantee::exact("exact"))))) - .is_some_and(|g| g.is_exact()) - ); - } - - /// Every candidate of the Stage 1 to Stage 3 run of one query. - fn run( - query: &str, - target: &AccuracyTarget, - data: &DataWorkload, - models: SelectionModels<'_>, - ) -> (Vec, WholePlanSelection) { - run_selecting(query, target, target, data, models) - } - - /// As [`run`], with summaries sized for `planned` and outputs checked - /// against `selected`. - fn run_selecting( - query: &str, - planned: &AccuracyTarget, - selected: &AccuracyTarget, - data: &DataWorkload, - models: SelectionModels<'_>, - ) -> (Vec, WholePlanSelection) { - let root = lower_promql(query, planned.clone()); - let entries = [entry_with(selected.clone(), QueryRecurrence::Unknown)]; - let roots = sized_as_pass1( - &[QueryRoot::Operator(root)], - std::slice::from_ref(planned), - data, - ); - let candidates = exhaustive_candidates(&roots, &entries, data); - let selection = select_among_plans(&candidates, &entries, data, models).unwrap(); - (candidates, selection) - } - - /// Every whole plan of `roots`: each combination of Pass 1's choices - /// composed, then every materialization assignment Stage 2 offers for - /// it, with unique ids. - fn exhaustive_candidates( - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - ) -> Vec { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(roots), - ) - .unwrap(); - let mut choices = vec![vec![]]; - for target in &inventory.targets { - choices = choices - .into_iter() - .flat_map(|choice: Vec| { - (0..target.choice_count()).map(move |index| { - let mut choice = choice.clone(); - choice.push(index); - choice - }) - }) - .collect(); - } - let mut candidates = Vec::new(); - for (index, choice) in choices.iter().enumerate() { - let Ok(composed) = compose_logical_candidate(&inventory, choice) else { - continue; - }; - let id = format!("L{}", index + 1); - let Ok(physical) = optimize_physical_asap(&id, &composed, entries, data, false) else { - continue; - }; - for (k, mut plan) in physical.into_iter().enumerate() { - plan.id = format!("P{}-{k}", index + 1); - plan.from_logical = id.clone(); - candidates.push(plan); - } - } - candidates - } - - fn sketches(candidate: &PhysicalWorkloadPlan) -> Vec { - candidate - .dag - .nodes - .iter() - .filter_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(*kind.algorithm()), - _ => None, - }) - .collect() - } - - fn reads_only(candidate: &PhysicalWorkloadPlan, algorithm: SketchAlgorithm) -> bool { - let sketches = sketches(candidate); - !sketches.is_empty() && sketches.iter().all(|a| *a == algorithm) - } - - fn output_guarantee(candidate: &PhysicalWorkloadPlan) -> Option { - root_guarantee(&candidate.roots[0]) - } - - /// `max by (job)` over per-series DDSketch quantiles keeps their relative - /// bound, and `2 · x` keeps it too (scaling preserves a relative error); - /// so those candidates meet the target. Over KLL (a rank error) the - /// result is a rank interval: no guarantee in the MVP. - #[test] - fn extremum_and_scaling_over_per_series_estimates_compose() { - let target = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - for query in [ - "max by (job) (quantile_over_time(0.99, m[1m]))", - "2 * max by (job) (quantile_over_time(0.99, m[1m]))", - ] { - let (candidates, selection) = run( - query, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty(), "{query}"); - for candidate in ddsketch { - let g = bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin(), - ) - .expect(query); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative, "{query}"); - assert!(g.spec.error.evaluate().unwrap() <= 0.01, "{query}"); - assert!(selection.costs.contains_key(&candidate.id), "{query}"); - } - for candidate in candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::Kll)) - { - assert_eq!( - bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin() - ), - None, - "{query}" - ); - } - } - } - - /// Negation, `abs` and `clamp` keep a DDSketch quantile's relative bound, - /// so those candidates are selectable; a shift by a constant, a - /// comparison (filter or `bool`) over the estimate compose to no - /// guarantee in the MVP, so those are not. - #[test] - fn lipschitz_functions_keep_and_shifts_and_comparisons_drop_a_relative_bound() { - let target = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let q = "quantile_over_time(0.99, m[1m])"; - for (query, kept) in [ - (format!("-{q}"), true), - (format!("abs({q})"), true), - (format!("clamp({q}, 1, 100)"), true), - (format!("clamp_max({q}, 100)"), true), - (format!("{q} + 1"), false), - (format!("{q} > 5"), false), - (format!("{q} > bool 5"), false), - ] { - let (candidates, selection) = run( - &query, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty(), "{query}"); - for candidate in ddsketch { - let g = bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin(), - ); - if kept { - let g = g.expect(&query); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative, "{query}"); - assert!(g.spec.error.evaluate().unwrap() <= 0.01, "{query}"); - assert!(selection.costs.contains_key(&candidate.id), "{query}"); - } else { - assert_eq!(g, None, "{query}"); - assert!(!selection.costs.contains_key(&candidate.id), "{query}"); - } - } - } - } - - /// A time-window fold over a subquery of estimates folds one estimate - /// per subquery step and series, so its failure probability is scaled - /// by the step count `⌈range / resolution⌉` (here 10), not by the input - /// series count. Without an explicit resolution the step count is not in - /// the query, and the fold has no guarantee. - #[test] - fn a_fold_over_subquery_estimates_counts_its_steps() { - let target = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let q = "quantile_over_time(0.99, m[1m])"; - let (candidates, selection) = run( - &format!("max_over_time({q}[10m:1m])"), - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty()); - for candidate in ddsketch { - let g = bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin(), - ) - .expect("a guarantee"); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(g.spec.error.evaluate().unwrap() <= 0.01); - let ProbabilityExpr::Scaled { count, .. } = &g.spec.failure_probability else { - panic!("{g:?}"); - }; - assert_eq!(count, &BoundExpr::Constant { value: 10.0 }); - assert!(selection.costs.contains_key(&candidate.id)); - } - let (candidates, _) = run( - &format!("max_over_time({q}[10m:])"), - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - for candidate in candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - { - assert_eq!( - bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin() - ), - None - ); - } - } - - /// Counting estimates by a label keeps the count exact: the group keys - /// are labels, not the estimates (#669). - #[test] - fn counting_estimates_by_a_label_stays_exact() { - let target = AccuracyTarget::max_error(0.01); - let (candidates, _) = run( - "count by (le) (quantile_over_time(0.99, m[1m]))", - &target, - &data(), - SelectionModels::builtin(), - ); - // The inner quantile is a sketch; the count itself is exact. - let sketched: Vec<_> = candidates - .iter() - .filter(|c| { - let sketches = sketches(c); - !sketches.is_empty() - && sketches - .iter() - .all(|a| matches!(a, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch)) - }) - .collect(); - assert!(!sketched.is_empty()); - for candidate in sketched { - assert!( - output_guarantee(candidate).is_some_and(|g| g.is_exact()), - "{:?}", - sketches(candidate) - ); - } - } - - /// `avg by (job)` over per-series DDSketch quantiles has a relative bound - /// that needs the values' sign, a statistic no input supplies: it stays - /// symbolic and plan selection treats it as unmet. - #[test] - fn average_over_relative_estimates_needs_the_sign() { - use asap_logical_asap_optimization::test_support as pcc; - use asap_types::stages::HowToComputeAggregateMeasure; - let target = AccuracyTarget::max_error(0.01); - let root = lower_promql( - "avg by (job) (quantile_over_time(0.99, m[1m]))", - target.clone(), - ); - let inventory = pcc::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - // Exercise the intended approximation directly; a joint optimizer may - // legitimately prefer an exact implementation or a different sharing - // topology, neither of which tests this compositional error contract. - let choices: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - *candidate - == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch) - }) - .unwrap_or(0) - }) - .collect(); - assert!(inventory - .targets - .iter() - .zip(&choices) - .any(|(target, &choice)| target.ways_to_compute[choice] - == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch))); - let outputs = pcc::compose_logical_candidate(&inventory, &choices).unwrap(); - let output = &outputs[0]; - let guarantee = root_guarantee(output).expect("a symbolic bound"); - assert!(guarantee.spec.error.has_unknown(), "{guarantee:?}"); - assert!(guarantee_check(0, output, &target, &data(), &SelectionModels::builtin()).is_err()); - } - - /// `max` folds one value per series, so its failure probability is - /// scaled by the symbolic input row count. Stage 3 binds it from the - /// series count of the source the query reads; without one, the target - /// is unmet. - #[test] - fn stage3_binds_the_input_row_count_from_the_data_workload() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Relative); - let query = "max by (job) (quantile_over_time(0.99, m[1m]))"; - let ddsketch = |candidates: &[PhysicalWorkloadPlan]| -> Vec { - candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .map(|c| c.id.clone()) - .collect() - }; - let (candidates, with) = run( - query, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ids = ddsketch(&candidates); - assert!(!ids.is_empty()); - assert!(ids.iter().all(|id| with.costs.contains_key(id))); - let mut unknown = with_series(None); - for facts in unknown.sources.values_mut() { - facts.values_non_negative = true; - } - // Evaluate the same physical candidates: IDs from independent timed - // searches do not identify the same implementation or connections. - let without = select_among_plans( - &candidates, - &[entry_with(target, QueryRecurrence::Unknown)], - &unknown, - SelectionModels::builtin(), - ) - .unwrap(); - assert!(ids.iter().all(|id| !without.costs.contains_key(id))); - } - - /// Deployment contract: every operand lies in `[1, 1000]`. - struct BoundedOperands; - - impl AccuracyEvidenceProvider for BoundedOperands { - fn operand_domain(&self, _operand: &OperatorNode) -> Option { - Some(asap_accuracy::OperandDomain { - lower: 1.0, - upper: 1_000.0, - }) - } - } - - /// A ratio of two DDSketch quantiles has a closed-form relative bound, - /// `(a + b)/(1 − b)`, with no operand domain: each operand sized for - /// 0.01 gives about 0.0202, which meets a 0.05 target. - #[test] - fn ddsketch_ratio_meets_its_target_without_operand_domains() { - let planned = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let query = "quantile_over_time(0.99, m[1m]) / quantile_over_time(0.5, m[1m])"; - let (candidates, selection) = run_selecting( - query, - &planned, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ids: Vec<_> = candidates - .iter() - .filter(|c| sketches(c) == [SketchAlgorithm::DDSketch, SketchAlgorithm::DDSketch]) - .map(|c| c.id.clone()) - .collect(); - // The Pass 1 candidate and its Pass 2 candidates sharing the input. - assert!(!ids.is_empty()); - assert!( - ids.iter().all(|id| selection.costs.contains_key(id)), - "{selection:?}" - ); - } - - /// `query` carrying `guarantee`, as a `scalar(…)` operand. - fn scalar_of(query: &str, guarantee: AccuracyGuarantee) -> ScalarExpr { - let node = lower_promql(query, AccuracyTarget::exact()); - ScalarExpr::PromqlScalarFromVector(Rc::new((*node).clone().with_guarantee(Some(guarantee)))) - } - - fn guarantee(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - metric, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - - fn scalar_root(op: ArithmeticOpKind, left: ScalarExpr, right: ScalarExpr) -> QueryRoot { - QueryRoot::Scalar(ScalarExpr::Arithmetic { - op, - left: Box::new(left), - right: Box::new(right), - semantics: ExprSemantics::Promql, - }) - } - - fn norms(sources: &[(&str, f64)]) -> DataWorkload { - let mut data = data(); - for (source, l1) in sources { - data.sources - .entry(source.to_string()) - .or_default() - .stream_norms = Some(asap_types::workload::StreamNorms { - l1: Some(*l1), - l2: None, - }); - } - data - } - - /// A difference of two Count-Min estimates of the raw values of - /// different metrics is bound by each metric's own stream norm from the - /// data workload - /// (`0.01 · 100 + 0.02 · 1000`); a metric without a norm leaves the - /// bound unmet. - #[test] - fn stage3_binds_each_metric_stream_norm() { - // Count-Min summaries of each metric's raw value column. - let raw = |query: &str, g: AccuracyGuarantee| { - ScalarExpr::PromqlScalarFromVector( - asap_logical_asap_optimization::test_support::estimate_over( - asap_logical_asap_optimization::test_support::weighted( - asap_logical_asap_optimization::test_support::cms_summary(query), - asap_types::ir::schema::SummaryInputExpr::Column( - asap_types::ir::scalar::ColumnRef::SampleValue, - ), - ), - g, - ), - ) - }; - let root = scalar_root( - ArithmeticOpKind::Sub, - raw( - "count by (job) (a)", - guarantee(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01, 0.001), - ), - raw( - "count by (job) (b)", - guarantee(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.02, 0.001), - ), - ); - let models = SelectionModels::builtin(); - let both = bound_of(&root, &norms(&[("a", 100.0), ("b", 1_000.0)]), &models).unwrap(); - assert!( - (both.spec.error.evaluate().unwrap() - 21.0).abs() < 1e-9, - "{both:?}" - ); - let one = bound_of(&root, &norms(&[("a", 100.0)]), &models).unwrap(); - assert!(one.spec.error.has_unknown()); - } - - /// A stream norm is bound only for a summary of the source's raw value - /// column: a count over the same source (constant weight) needs the - /// source's row count, which nothing binds, and a summary of a computed - /// value (`m * 1000`) needs a norm nothing binds; both stay unmet. - #[test] - fn stage3_binds_a_stream_norm_only_for_the_raw_value_column() { - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::SummaryInputExpr; - let frequency = guarantee(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01, 0.001); - let doubled = |summary: Rc| { - scalar_root( - ArithmeticOpKind::Mul, - ScalarExpr::literal_f64(2.0), - ScalarExpr::PromqlScalarFromVector( - asap_logical_asap_optimization::test_support::estimate_over( - summary, - frequency.clone(), - ), - ), - ) - }; - let models = SelectionModels::builtin(); - let data = norms(&[("m", 100.0)]); - let column = || SummaryInputExpr::Column(ColumnRef::SampleValue); - let raw = doubled(asap_logical_asap_optimization::test_support::weighted( - asap_logical_asap_optimization::test_support::cms_summary("count by (job) (m)"), - column(), - )); - let g = bound_of(&raw, &data, &models).unwrap(); - assert!( - (g.spec.error.evaluate().unwrap() - 2.0).abs() < 1e-12, - "{g:?}" - ); - for unbound in [ - doubled(Rc::new( - asap_logical_asap_optimization::test_support::cms_summary("count by (job) (m)"), - )), - doubled(asap_logical_asap_optimization::test_support::weighted( - asap_logical_asap_optimization::test_support::cms_summary( - "count by (job) (m * 1000)", - ), - column(), - )), - ] { - let g = bound_of(&unbound, &data, &models).unwrap(); - assert!(g.spec.error.has_unknown(), "{g:?}"); - } - } - - /// A product and a quotient of two estimates are bound from the - /// deployment's operand domains (`[1, 1000]`): - /// `|x|·e_y + |y|·e_x + e_x·e_y` and `(|x|·e_y + |y|·e_x) / (|y|·(|y| − e_y))` - /// at the domain's extremes; without domains they stay unmet. A ratio of - /// relative errors needs no domain: `(a + b)/(1 − b)`. - #[test] - fn stage3_binds_products_and_quotients_from_operand_domains() { - let (x, y) = ( - scalar_of("sum(a)", guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001)), - scalar_of("sum(b)", guarantee(AccuracyMetric::ABSOLUTE, 0.02, 0.001)), - ); - let product = scalar_root(ArithmeticOpKind::Mul, x.clone(), y.clone()); - let quotient = scalar_root(ArithmeticOpKind::Div, x, y); - let bounded = SelectionModels::builtin().with_evidence(&BoundedOperands); - for (root, expected) in [ - (&product, 1_000.0 * 0.02 + 1_000.0 * 0.01 + 0.01 * 0.02), - ("ient, (1_000.0 * 0.02 + 1_000.0 * 0.01) / (1.0 - 0.02)), - ] { - let g = bound_of(root, &data(), &bounded).unwrap(); - assert!( - (g.spec.error.evaluate().unwrap() - expected).abs() < 1e-9, - "{g:?}" - ); - assert!((g.spec.failure_probability.evaluate().unwrap() - 0.002).abs() < 1e-12); - let unbound = bound_of(root, &data(), &SelectionModels::builtin()).unwrap(); - assert!(unbound.spec.error.has_unknown(), "{unbound:?}"); - } - let ratio = scalar_root( - ArithmeticOpKind::Div, - scalar_of("sum(a)", guarantee(AccuracyMetric::Relative, 0.01, 0.0)), - scalar_of("sum(b)", guarantee(AccuracyMetric::Relative, 0.01, 0.0)), - ); - let g = bound_of(&ratio, &data(), &SelectionModels::builtin()).unwrap(); - assert!( - (g.spec.error.evaluate().unwrap() - 0.02 / 0.99).abs() < 1e-12, - "{g:?}" - ); - } - - /// A ratio divided by an exact vector, `(q99 / q50) / max_over_time(m)`, - /// keeps the inner ratio's relative bound, so the candidate is - /// selectable with no operand domain. - #[test] - fn a_scaled_ratio_keeps_its_relative_bound() { - let planned = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let query = "(quantile_over_time(0.99, m[1m]) / quantile_over_time(0.5, m[1m])) \ - / max_over_time(m[1m])"; - let (candidates, selection) = run_selecting( - query, - &planned, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ids: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .map(|c| c.id.clone()) - .collect(); - assert!(!ids.is_empty()); - assert!( - ids.iter().all(|id| selection.costs.contains_key(id)), - "{selection:?}" - ); - } - - /// The input row count is bound only from the series count of the - /// sources the query reads: another source's declared series count - /// leaves it unbound. - #[test] - fn stage3_binds_only_the_series_count_of_the_sources_read() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let query = "max by (job) (quantile_over_time(0.99, m[1m]))"; - let mut other_source = data(); - other_source.sources.remove("m"); - let (candidates, selection) = - run(query, &target, &other_source, SelectionModels::builtin()); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty()); - assert!(ddsketch - .iter() - .all(|c| !selection.costs.contains_key(&c.id))); - } - - /// `quantile_over_time(0.99, m[1m])` with its KLL candidate chosen, its - /// state materialized as `materialization`. - fn kll_dag(materialization: Materialization) -> PhysicalASAPDAG { - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - let root = lower_promql( - "quantile_over_time(0.99, m[1m])", - AccuracyTarget::max_error(0.01).failure_probability(0.01), - ); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let choice: Vec<_> = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(|c| { - *c == asap_types::stages::HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::Kll, - ) - }) - .unwrap_or(0) - }) - .collect(); - let roots = asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - let QueryRoot::Operator(root) = &roots[0] else { - panic!("operator root") - }; - let state = OperatorNode::reachable(root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("a KLL state"); - let mut assignment = - MaterializationAssignment::with_default(Materialization::NotMaterialized); - assignment.set(&state, materialization); - let timed = - apply_materialization_timings(root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids(&[ - QueryRoot::Operator(timed), - ]) - .unwrap() - .dag - } - - /// The physical DAG of `query`'s first per-computation candidates, - /// with nothing materialized. - fn first_candidate_dag(query: &str) -> PhysicalASAPDAG { - let root = lower_promql(query, AccuracyTarget::exact()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let choice = vec![0; inventory.targets.len()]; - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - optimize_physical_asap("L1", &roots, &[], &data(), false) - .unwrap() - .remove(0) - .dag - } - - /// A time shift is free and does not change what the scan below it - /// reads: the scan reads the window above the shift, as without it. A - /// time range is priced by the rows it keeps. - #[test] - fn time_shift_is_free_and_the_scan_reads_the_shifted_window() { - let every_minute = [repeating(60_000)]; - let plain = first_candidate_dag("sum_over_time(m[5m])"); - let shifted = first_candidate_dag("sum_over_time(m[5m] offset 1h)"); - let (plain_cost, shifted_cost) = ( - priced(&plain, &every_minute), - priced(&shifted, &every_minute), - ); - let cost_of = |dag: &PhysicalASAPDAG, cost: &PlanCost, pick: fn(&Payload) -> bool| { - cost.per_node[&node_of(dag, pick)].cost_per_second.unwrap() - }; - let is_shift = |p: &Payload| matches!(p, Payload::NonASAP(NonASAPOp::TimeShift { .. })); - let is_range = |p: &Payload| matches!(p, Payload::NonASAP(NonASAPOp::TimeRange { .. })); - assert_eq!(cost_of(&shifted, &shifted_cost, is_shift), 0.0); - assert!(close( - cost_of(&shifted, &shifted_cost, is_scan), - cost_of(&plain, &plain_cost, is_scan) - )); - assert!(close( - shifted_cost.cost_per_second.unwrap(), - plain_cost.cost_per_second.unwrap() - )); - let range = &plain_cost.per_node[&node_of(&plain, is_range)]; - assert!(range.detail.contains("keep"), "{}", range.detail); - } - - const AT_INGESTION: Materialization = - Materialization::MatAtIngestion(asap_types::physical::Medium::Memory); - const AT_QUERY: Materialization = - Materialization::MatAtQuery(asap_types::physical::Medium::Memory); - - fn node_of(dag: &PhysicalASAPDAG, pick: impl Fn(&Payload) -> bool) -> PhysicalASAPNodeId { - dag.nodes.iter().find(|n| pick(&n.payload)).unwrap().id - } - - fn is_scan(payload: &Payload) -> bool { - matches!(payload, Payload::NonASAP(NonASAPOp::Scan { .. })) - } - - fn is_build(payload: &Payload) -> bool { - matches!(payload, Payload::ASAP(ASAPOp::SummaryAgg { .. })) - } - - fn priced(dag: &PhysicalASAPDAG, recurrence: &[QueryRecurrence]) -> PlanCost { - price(dag, recurrence, &data(), &CostInputs::default()).unwrap() - } - - /// The default settings with cost model 1's unit costs edited. - fn unit_costs(edit: impl FnOnce(&mut ResourceUnitCost)) -> CostInputs { - let mut prices = ResourceUnitCost::illustrative(); - edit(&mut prices); - CostInputs { - cost_model: CostModel::UsageOverTime(prices), - ..CostInputs::default() - } - } - - fn close(a: f64, b: f64) -> bool { - (a - b).abs() <= 1e-12 * a.abs().max(b.abs()).max(1e-300) - } - - /// An ingestion-time node is priced over one second of arriving rows, - /// once per second, whatever its readers' evaluation rate; a query-time - /// node scales with that rate. - #[test] - fn ingestion_time_node_is_priced_per_second_of_arriving_rows() { - let dag = kll_dag(AT_INGESTION); - let (fast, slow) = ( - priced(&dag, &[repeating(1_000)]), - priced(&dag, &[repeating(10_000)]), - ); - for id in [node_of(&dag, is_scan), node_of(&dag, is_build)] { - assert_eq!( - fast.per_node[&id].cost_per_second, - slow.per_node[&id].cost_per_second - ); - assert!(fast.per_node[&id].detail.contains("; x 1.0000/s")); - } - let estimate = dag.roots[0]; - assert!(close( - fast.per_node[&estimate].cost_per_second.unwrap(), - 10.0 * slow.per_node[&estimate].cost_per_second.unwrap() - )); - } - - /// Kept state is charged per byte per second: at ingestion time, twice - /// the state's bytes (the window being built and the completed one) for - /// every second. At query time the state is reused only within one - /// evaluation, so it is held only while that evaluation runs: its bytes - /// times the evaluation's latency, once per evaluation. State computed - /// and dropped within a query (`NotMaterialized`) is held while that - /// query runs too, and charged the same way. - #[test] - fn kept_state_is_charged_per_byte_per_second() { - let cost = |dag: &PhysicalASAPDAG, w| { - let settings = unit_costs(|c| c.cost_per_memory_byte_second = w); - price(dag, &recurrences(1), &data(), &settings).unwrap() - }; - for materialization in [AT_INGESTION, AT_QUERY, Materialization::NotMaterialized] { - let dag = kll_dag(materialization); - let build = node_of(&dag, is_build); - let (with, without) = (cost(&dag, 1.25e-7), cost(&dag, 0.0)); - let Payload::ASAP(ASAPOp::SummaryAgg { family, .. }) = - &dag.nodes.iter().find(|n| n.id == build).unwrap().payload - else { - unreachable!() - }; - // One state per series: data() declares 10 000. At query time - // each sees its series' 12 samples: the 120 000 rows of its - // source's 1-min window (data()'s 10 000 rows per second over its - // five sources); at ingestion time its samples are unknown. - let samples = match materialization { - AT_INGESTION => StateInputRows::UNKNOWN, - _ => StateInputRows { - rows_per_state: Some(12), - non_negative: false, - }, - }; - let state = 10_000.0 * summary_shape(family, samples).1 as f64; - let byte_seconds = if materialization == AT_INGESTION { - 2.0 * state - } else { - let evaluation_s = with.usage.latency_per_query[0].total_ms() / 1_000.0; - assert!(evaluation_s > 0.0 && evaluation_s < 1.0, "{evaluation_s}"); - // Once per evaluation of the query repeating every 10 s. - assert!(with.per_node[&build].detail.contains("; x 0.1000/s")); - state * evaluation_s * 0.1 - }; - let term = with.per_node[&build].cost_per_second.unwrap() - - without.per_node[&build].cost_per_second.unwrap(); - assert!( - close(term, 1.25e-7 * byte_seconds), - "{materialization:?}: {term} vs {}", - 1.25e-7 * byte_seconds - ); - // The candidate's memory term covers every node's held memory, - // the build's included. - let candidate_term = with.cost_per_second.unwrap() - without.cost_per_second.unwrap(); - assert!(close( - candidate_term, - 1.25e-7 * with.usage.average_memory_bytes - )); - assert!(candidate_term >= term); - } - } - - /// Under cost model 1, memory is charged as bytes held per second - /// (average memory): the memory a computation holds while it runs (a - /// sketch being built, an exact quantile's values) counts for the time - /// it is held, once per evaluation. The memory price's term is exactly - /// the average memory, which is below the peak memory. - #[test] - fn memory_held_only_while_a_query_runs_is_charged_for_the_time_it_is_held() { - let memory_term = |dag: &PhysicalASAPDAG| { - let cost = |price_per_byte_second| { - let settings = - unit_costs(|c| c.cost_per_memory_byte_second = price_per_byte_second); - price(dag, &recurrences(1), &data(), &settings).unwrap() - }; - let (with, without) = (cost(1.0), cost(0.0)); - let term = with.cost_per_second.unwrap() - without.cost_per_second.unwrap(); - (term, with.usage) - }; - let mut dags: Vec<(String, PhysicalASAPDAG)> = [ - "quantile_over_time(0.99, m[1m])", - "quantile by (job) (0.99, m)", - ] - .into_iter() - .map(|query| { - let root = lower_promql(query, AccuracyTarget::exact()); - let dag = - optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data(), false) - .unwrap() - .remove(0) - .dag; - (query.to_string(), dag) - }) - .collect(); - for materialization in [AT_INGESTION, AT_QUERY, Materialization::NotMaterialized] { - dags.push((format!("{materialization:?}"), kll_dag(materialization))); - } - for (name, dag) in &dags { - let (term, usage) = memory_term(dag); - assert!( - close(term, usage.average_memory_bytes), - "{name}: {term} vs {}", - usage.average_memory_bytes - ); - assert!( - usage.peak_memory_bytes as f64 > usage.average_memory_bytes, - "{name}: {usage:?}" - ); - } - // Not materialized, nothing is kept, but the sketch being built is - // held while the query runs: the memory price adds to the cost. - let (term, _) = memory_term(&kll_dag(Materialization::NotMaterialized)); - assert!(term > 0.0, "{term}"); - } - - /// Kept exact state that grows with the data is charged by the values it - /// keeps: a materialized exact quantile keeps every value of its group, - /// a sum one value per group. - #[test] - fn kept_exact_state_is_charged_by_the_values_it_keeps() { - let kept = |query: &str| { - let root = lower_promql(query, AccuracyTarget::exact()); - let mut dag = - optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data(), false) - .unwrap() - .remove(0) - .dag; - let aggregate = dag - .nodes - .iter_mut() - .find(|n| matches!(n.payload, Payload::NonASAP(NonASAPOp::Aggregate { .. }))) - .unwrap(); - aggregate.materialization = AT_QUERY; - let id = aggregate.id; - let cost = |w| { - let settings = unit_costs(|c| c.cost_per_memory_byte_second = w); - // 10 jobs, so each group keeps many values. - let mut data = arriving(10_000.0); - data.sources.get_mut("m").unwrap().distinct_values_per_label = - [("job".to_string(), 10)].into(); - price(&dag, &recurrences(1), &data, &settings) - .unwrap() - .per_node[&id] - .cost_per_second - .unwrap() - }; - (cost(1.0) - cost(0.0)).round() as u64 - }; - let (quantile, sum) = ( - kept("quantile by (job) (0.99, m)"), - kept("sum by (job) (m)"), - ); - assert!(quantile > 10 * sum, "{quantile} vs {sum}"); - } - - /// A `NotMaterialized` node read by two queries is computed for each, at - /// each one's rate; `MatAtQuery`, it is computed once at their combined - /// rate, where queries at the same interval run together. - #[test] - fn rates_follow_per_consumer_charging() { - let root = lower_promql("sum by (job) (rate(m[1m]))", AccuracyTarget::exact()); - let one = optimize_physical_asap( - "L", - &[QueryRoot::Operator(root.clone())], - &[], - &data(), - false, - ) - .unwrap() - .remove(0) - .dag; - let mut two = optimize_physical_asap( - "L", - &[QueryRoot::Operator(root.clone()), QueryRoot::Operator(root)], - &[], - &data(), - false, - ) - .unwrap() - .remove(0) - .dag; - let without_memory = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let total = |dag: &PhysicalASAPDAG, recurrence: &[QueryRecurrence]| { - price(dag, recurrence, &data(), &without_memory) - .unwrap() - .cost_per_second - .unwrap() - }; - let single = total(&one, &recurrences(1)); - let mixed = [repeating(10_000), repeating(20_000)]; - assert!(close(total(&two, &recurrences(2)), 2.0 * single)); - assert!(close(total(&two, &mixed), 1.5 * single)); - assert_eq!(two.roots[0], two.roots[1], "one root node for both queries"); - for node in &mut two.nodes { - node.materialization = AT_QUERY; - } - assert!(close(total(&two, &recurrences(2)), single)); - assert!(close(total(&two, &mixed), 1.5 * single)); - } - - /// Queries that refresh together at one interval and step evaluate at - /// the same instants: a node they share runs at the most steps among - /// them, not their sum. One over the last hour (61 steps) and one over - /// the last 10 min (11 steps), both every minute at 1-min steps, and a - /// third with no range (its one evaluation is the others' last step): - /// 61 evaluations a minute. A different step adds only the instants - /// the others do not evaluate at. - #[test] - fn readers_at_one_interval_and_step_share_their_steps() { - let every_minute = repeating(60_000); - let range = |span_ms: u64, step_ms: u64| Some(RangeEvaluation { step_ms, span_ms }); - let rate = |queries: &[Option]| { - evaluation_rate(queries.iter().map(|r| (&every_minute, *r)), 3_600.0).unwrap() - }; - let close = |a: f64, b: f64| (a - b).abs() < 1e-12; - assert!(close(rate(&[range(3_600_000, 60_000)]), 61.0 / 60.0)); - assert!(close( - rate(&[range(3_600_000, 60_000), range(600_000, 60_000), None]), - 61.0 / 60.0 - )); - assert!(close(rate(&[None, None]), 1.0 / 60.0)); - // A one-step range evaluates only at the request's end. - assert!(close( - rate(&[range(3_600_000, 60_000), range(0, 30_000)]), - 61.0 / 60.0 - )); - // A 30 s step over the last 10 min shares 11 of its 21 instants - // with the 1-min steps over the last hour: 71 distinct instants. - assert!(close( - rate(&[range(3_600_000, 60_000), range(600_000, 30_000)]), - 71.0 / 60.0 - )); - } - - /// Queries every 20 s at phases 0 and 5 s evaluate at different - /// instants: a node they share runs at both, twice as often as at one - /// phase. Phases 0 and 40 s are one phase on 20 s, and a query of - /// unknown phase runs with the known one. - #[test] - fn readers_at_different_phases_evaluate_apart() { - let at = |phase_ms: u64| { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: asap_types::workload::RepetitionInterval(20_000), - evaluation_phase: asap_types::workload::TimestampMs(phase_ms), - }) - }; - let unknown = repeating(20_000); - let rate = |queries: &[&QueryRecurrence]| { - evaluation_rate(queries.iter().map(|r| (*r, None)), 3_600.0).unwrap() - }; - let close = |a: f64, b: f64| (a - b).abs() < 1e-12; - assert!(close(rate(&[&at(0), &at(5_000)]), 2.0 / 20.0)); - assert!(close(rate(&[&at(0), &at(40_000)]), 1.0 / 20.0)); - assert!(close(rate(&[&at(5_000), &unknown]), 1.0 / 20.0)); - assert!(close(rate(&[&at(0), &at(5_000), &unknown]), 2.0 / 20.0)); - } - - /// One-off and unknown recurrence are spread over the horizon: - /// invocations × per-evaluation cost / non_repeating_query_horizon_seconds. - #[test] - fn one_off_recurrence_is_spread_over_the_horizon() { - let root = lower_promql("sum by (job) (rate(m[1m]))", AccuracyTarget::exact()); - let dag = optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data(), false) - .unwrap() - .remove(0) - .dag; - let horizon = CostInputs::default().non_repeating_query_horizon_seconds; - assert_eq!(horizon, 3_600.0); - let per_evaluation = priced(&dag, &[repeating(1_000)]).cost_per_second.unwrap(); - let once = priced( - &dag, - &[QueryRecurrence::OneTime { - invocations: 3, - execute_at: None, - }], - ) - .cost_per_second - .unwrap(); - assert!(close(once, 3.0 * per_evaluation / horizon), "{once}"); - assert!(close( - priced(&dag, &[QueryRecurrence::Unknown]) - .cost_per_second - .unwrap(), - per_evaluation / horizon - )); - } - - /// Settings outside their domain are an error, not a price, checked once - /// before any candidate: a price the cost model reads that is negative, - /// the model's prices all 0 (cost model 1 with only provisioned prices - /// set), or a horizon that is not positive. A price the model does not - /// read is not checked. - #[test] - fn invalid_settings_are_rejected() { - let candidates = tumbling_window_candidates(10_000.0); - let select = |settings: &CostInputs| { - select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(settings), - ) - }; - let only_provisioned = ResourceUnitCost { - cost_per_cpu_op: 0.0, - cost_per_scanned_byte: 0.0, - cost_per_memory_byte_second: 0.0, - cost_per_disk_byte_second: 0.0, - ..ResourceUnitCost::illustrative() - }; - for settings in [ - unit_costs(|c| c.cost_per_memory_byte_second = -1.0), - under(CostModel::UsageOverTime(only_provisioned.clone())), - CostInputs { - non_repeating_query_horizon_seconds: 0.0, - ..CostInputs::default() - }, - ] { - assert!( - matches!( - select(&settings), - Err(ErrorSelectingPlan::InvalidSettings(_)) - ), - "{settings:?}" - ); - } - assert!(select(&under(CostModel::ProvisionedCapacity(only_provisioned))).is_ok()); - assert!(select(&unit_costs( - |c| c.cost_per_provisioned_cpu_core_second = -1.0 - )) - .is_ok()); - } - - // ── Stage 2 materialization and the latency check ──────────────────── - - const MINUTE: u32 = 60_000; - - /// The p99 target [`tumbling_window_candidates`] are sized for. - fn p99_target() -> AccuracyTarget { - AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank) - } - - /// A predictable p99 query evaluated every minute, with no latency bound. - fn every_minute() -> QueryWorkloadEntry { - QueryWorkloadEntry { - predictability: asap_types::workload::Predictability::Predictable { known_at: None }, - ..entry_with(p99_target(), repeating(MINUTE)) - } - } - - /// [`data`] with each test source arriving at `rows_per_s`: pricing - /// gives each source its series share of the workload's rate. - fn arriving(rows_per_s: f64) -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: rows_per_s * TEST_SOURCES.len() as f64, - }, - ..data() - } - } - - /// The physical candidates of a per-series p99 over 5 min with its KLL - /// in five 1-min tumbling windows, every minute over data arriving at - /// `rows_per_s`: rebuilt at query time (B2), then maintained at - /// ingestion time (B1). - fn tumbling_window_candidates(rows_per_s: f64) -> Vec { - let candidates = - tumbling_window_candidates_for(rows_per_s, &DeploymentCapabilities::default()); - assert_eq!(candidates.len(), 2); - candidates - } - - /// [`tumbling_window_candidates`] for a deployment with `deployment`'s - /// capabilities: B2, B1, then, when it keeps query-time results, B3 - /// (built at query time and kept across evaluations). - fn tumbling_window_candidates_for( - rows_per_s: f64, - deployment: &DeploymentCapabilities, - ) -> Vec { - let root = lower_promql("quantile_over_time(0.99, m[5m])", p99_target()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let kll = inventory.targets[0] - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - asap_types::stages::HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap(); - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, - &[kll], - ) - .unwrap(); - let roots = sized_as_pass1(&roots, &[p99_target()], &arriving(rows_per_s)); - let rule_candidates = - asap_logical_asap_optimization::test_support::tumbling_window_candidates( - &roots, - &[Some(u64::from(MINUTE))], - ); - let roots = - asap_logical_asap_optimization::test_support::window_composition_pass2_candidates( - &roots, - &rule_candidates, - ) - .unwrap() - .pop() - .unwrap() - .roots - .unwrap(); - let mut candidates = optimize_physical_asap( - "L1", - &roots, - &[every_minute()], - &arriving(rows_per_s), - deployment.query_time_results_retained, - ) - .unwrap(); - for (k, c) in candidates.iter_mut().enumerate() { - c.id = format!("P1-m{k}"); - } - candidates - } - - /// The raw data samples the scan `scan` reads per computation, from its - /// priced detail (`scan samples; ...`). - fn scanned_samples(cost: &PlanCost, scan: PhysicalASAPNodeId) -> u64 { - let detail = &cost.per_node[&scan].detail; - detail - .strip_prefix("scan ") - .and_then(|rest| rest.split(' ').next()) - .and_then(|rows| rows.parse().ok()) - .unwrap_or_else(|| panic!("not a scan: {detail}")) - } - - /// A deployment that keeps query-time results across evaluations. - fn keeping_query_time_results() -> DeploymentCapabilities { - DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - } - } - - /// `query_time_results_retained` gates keeping a query-time result across - /// evaluations, and only that. Without it, Stage 2 offers the 1-min - /// tumbling windows of a query repeating every minute rebuilt (B2) or maintained - /// (B1); with it, also built at query time and kept (B3, - /// `MatAtQueryKeptAcrossEvaluations`). B3 is computed once per - /// evaluation, reads one tumbling window (1 min) of raw data, keeps that much raw - /// data on disk, and holds its 4 older tumbling windows in memory between - /// evaluations (on disk, if it were placed there). The same chain - /// `MatAtQuery` is reused only within one evaluation: the scan reads the - /// 5-min window and nothing is kept between evaluations. - #[test] - fn query_time_results_retained_keeps_a_tumbling_window_chain_across_evaluations() { - assert_eq!(tumbling_window_candidates(10_000.0).len(), 2); - let candidates = tumbling_window_candidates_for(10_000.0, &keeping_query_time_results()); - assert_eq!(candidates.len(), 3); - let b2 = &candidates[0].dag; - let b3 = &candidates[2].dag; - let chain = node_of(b3, is_chain); - let with_chain = |materialization: Materialization| { - let mut dag = b3.clone(); - for node in dag.nodes.iter_mut().filter(|n| n.id == chain) { - node.materialization = materialization; - } - dag - }; - assert_eq!( - *b3, - with_chain(Materialization::MatAtQueryKeptAcrossEvaluations( - Medium::Memory - )) - ); - let recurrence = [repeating(MINUTE)]; - let settings = CostInputs::default(); - let kept = price(b3, &recurrence, &arriving(10_000.0), &settings).unwrap(); - let rebuilt = price(b2, &recurrence, &arriving(10_000.0), &settings).unwrap(); - let scan = node_of(b3, is_scan); - assert_eq!(scanned_samples(&kept, scan), 10_000 * 60); - assert_eq!( - scanned_samples(&rebuilt, node_of(b2, is_scan)), - 10_000 * 300 - ); - assert_eq!(kept.usage.disk_bytes * 5, rebuilt.usage.disk_bytes); - let Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { family, .. }) = - &b3.nodes.iter().find(|n| n.id == chain).unwrap().payload - else { - unreachable!() - }; - // Each tumbling window sees its series' 60 samples: the one minute of 10 000 - // rows per second over 10 000 series it reads. - let samples = StateInputRows { - rows_per_state: Some(60), - non_negative: false, - }; - let (_, state_bytes) = summary_shape(family, samples); - let chain_cost = &kept.per_node[&chain]; - assert!(chain_cost.detail.contains("; x 0.0167/s"), "{chain_cost:?}"); - // One state per series: arriving() declares 10 000. - let between = 4 * 10_000 * state_bytes; - assert!(chain_cost - .detail - .contains(&format!("keeps {between} bytes across evaluations"))); - // On disk, the same N − 1 older tumbling windows are kept. - let on_disk = with_chain(Materialization::MatAtQueryKeptAcrossEvaluations( - Medium::Disk, - )); - let on_disk = price(&on_disk, &recurrence, &arriving(10_000.0), &settings).unwrap(); - assert_eq!( - on_disk.usage.disk_bytes, - kept.usage.disk_bytes + between, - "{on_disk:?}" - ); - // Shared within one evaluation, nothing is kept between them. - let shared = with_chain(Materialization::MatAtQuery(Medium::Memory)); - let within_one = price(&shared, &recurrence, &arriving(10_000.0), &settings).unwrap(); - assert_eq!(scanned_samples(&within_one, scan), 10_000 * 300); - assert!(!within_one.per_node[&chain] - .detail - .contains("across evaluations")); - assert!(within_one.usage.average_memory_bytes < kept.usage.average_memory_bytes); - } - - fn is_chain(payload: &Payload) -> bool { - matches!( - payload, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - } - - /// #509 Example 4, B1: maintained at ingestion time, a chain of N tumbling windows - /// pays the update of the newest tumbling window only, one second of arriving rows - /// once per second, and keeps N + 1 tumbling windows (the N the query merges and - /// the one being built) per series. - #[test] - fn a_chain_at_ingestion_pays_the_newest_tumbling_window_and_keeps_n_plus_one_tumbling_windows() - { - let candidates = tumbling_window_candidates(10_000.0); - let dag = &candidates[1].dag; - let chain = node_of(dag, is_chain); - let Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_count, - .. - }) = &dag.nodes.iter().find(|n| n.id == chain).unwrap().payload - else { - unreachable!() - }; - assert_eq!(*tumbling_window_count, 5); - // Each tumbling window sees its series' 60 samples: one minute of 10 000 rows - // per second over 10 000 series. - let samples = StateInputRows { - rows_per_state: Some(60), - non_negative: false, - }; - let (depth, state_bytes) = summary_shape(family, samples); - let recurrence = [repeating(MINUTE)]; - let without_memory = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let work = &price(dag, &recurrence, &arriving(10_000.0), &without_memory) - .unwrap() - .per_node[&chain]; - assert!(work.detail.contains("; x 1.0000/s"), "{work:?}"); - assert!( - close( - work.cost_per_second.unwrap(), - 10_000.0 * (depth + 1.0) * 1e-6 - ), - "{work:?}" - ); - let with_memory = &price( - dag, - &recurrence, - &arriving(10_000.0), - &CostInputs::default(), - ) - .unwrap() - .per_node[&chain]; - let kept = 6 * 10_000 * state_bytes; - assert!( - close( - with_memory.cost_per_second.unwrap() - work.cost_per_second.unwrap(), - 1.25e-7 * kept as f64 - ), - "{with_memory:?}" - ); - assert!(with_memory.detail.contains(&format!("keeps {kept} bytes"))); - } - - /// The latency check (S6, sketch-bench cost model §5): a query's - /// latency is its query-time work on one core. Rebuilding the tumbling windows at - /// query time (B2) takes longer than merging the maintained ones (B1); - /// with a bound between the two, B2 is rejected as invalid with its - /// latency and B1 is selected. - #[test] - fn a_candidate_over_a_querys_latency_bound_is_rejected_with_its_latency() { - let candidates = tumbling_window_candidates(10_000.0); - let latency = |c: &PhysicalWorkloadPlan| { - let cost = price( - &c.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &CostInputs::default(), - ) - .unwrap(); - cost.usage.latency_per_query[0] - }; - let (b2, b1) = (latency(&candidates[0]), latency(&candidates[1])); - assert_eq!((b2.compaction_ms, b1.compaction_ms), (0.0, 0.0)); - assert!(b1.query_ms < b2.query_ms, "{b1:?} vs {b2:?}"); - let bound = (b1.query_ms + b2.query_ms) / 2.0; - let mut bounded = every_minute(); - bounded.requirements.max_latency_ms = Some(bound); - let selection = select_among_plans( - &candidates, - &[bounded], - &arriving(10_000.0), - SelectionModels::builtin(), - ) - .unwrap(); - assert_eq!(selection.selected, candidates[1].id); - let rejected = &selection.rejected[0]; - assert_eq!(rejected.id, candidates[0].id); - assert_eq!(rejected.cause, RejectionCause::LatencyBoundExceeded); - assert!( - rejected.reason.starts_with(&format!( - "q1: latency {:.1} ms per evaluation (compaction 0.0 ms + query", - b2.total_ms() - )) && rejected - .reason - .ends_with(&format!("is over its {bound} ms bound")), - "{}", - rejected.reason - ); - assert!(!selection.costs.contains_key(&candidates[0].id)); - } - - /// A query evaluated every minute whose evaluation takes longer than a - /// minute never keeps up, so even without a latency bound the - /// candidate is invalid. Over data fast enough that rebuilding the - /// tumbling windows at query time (B2) takes over a minute while merging the - /// maintained ones (B1) does not, B2 is rejected with its latency and - /// the interval, and B1 is selected. - #[test] - fn a_candidate_slower_than_a_querys_repeat_interval_is_rejected_without_a_bound() { - let rows_per_s = 1e9; - let candidates = tumbling_window_candidates(rows_per_s); - let latency = |c: &PhysicalWorkloadPlan| { - price( - &c.dag, - &[repeating(MINUTE)], - &arriving(rows_per_s), - &CostInputs::default(), - ) - .unwrap() - .usage - .latency_per_query[0] - }; - let (b2, b1) = (latency(&candidates[0]), latency(&candidates[1])); - assert!( - b1.total_ms() < f64::from(MINUTE) && b2.total_ms() > f64::from(MINUTE), - "{b1:?} vs {b2:?}" - ); - let unbounded = every_minute(); - assert_eq!(unbounded.requirements.max_latency_ms, None); - let selection = select_among_plans( - &candidates, - &[unbounded], - &arriving(rows_per_s), - SelectionModels::builtin(), - ) - .unwrap(); - assert_eq!(selection.selected, candidates[1].id); - let rejected = &selection.rejected[0]; - assert_eq!(rejected.id, candidates[0].id); - assert_eq!(rejected.cause, RejectionCause::LatencyOverRepeatInterval); - assert!(rejected.cause.failed_a_check()); - assert_eq!( - rejected.reason, - format!( - "q1: latency {:.1} ms per evaluation (compaction {:.1} ms + query {:.1} ms) is \ - over its {MINUTE} ms repeat interval, so its evaluations cannot keep up", - b2.total_ms(), - b2.compaction_ms, - b2.query_ms - ) - ); - assert!(!selection.costs.contains_key(&candidates[0].id)); - } - - /// With no repeat interval (a one-time query) there is nothing to keep - /// up with: the slow candidate stays valid and is selected when it is - /// cheaper. - #[test] - fn a_one_time_query_has_no_repeat_interval_to_keep_up_with() { - let rows_per_s = 1e9; - let candidates = tumbling_window_candidates(rows_per_s); - let once = QueryWorkloadEntry { - recurrence: QueryRecurrence::OneTime { - invocations: 1, - execute_at: None, - }, - ..every_minute() - }; - let selection = select_among_plans( - &candidates, - &[once], - &arriving(rows_per_s), - SelectionModels::builtin(), - ) - .unwrap(); - // Run once an hour, rebuilding at query time is the cheaper plan, - // over a minute per evaluation or not. - assert_eq!(selection.selected, candidates[0].id); - assert_eq!(selection.rejected.len(), 1, "{:?}", selection.rejected); - assert_eq!(selection.rejected[0].id, candidates[1].id); - assert_eq!(selection.rejected[0].cause, RejectionCause::Costlier); - } - - /// A query over both its latency bound and its repeat interval is - /// reported over its bound: the bound is the user's own requirement. - /// The interval is also checked for a repetition at a declared phase. - #[test] - fn a_broken_latency_bound_is_reported_before_the_repeat_interval() { - let rows_per_s = 1e9; - let candidates = tumbling_window_candidates(rows_per_s); - let mut bounded = QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: asap_types::workload::RepetitionInterval(MINUTE), - evaluation_phase: asap_types::workload::TimestampMs(0), - }), - ..every_minute() - }; - let select = |entry: &QueryWorkloadEntry| { - select_among_plans( - &candidates, - std::slice::from_ref(entry), - &arriving(rows_per_s), - SelectionModels::builtin(), - ) - .unwrap() - }; - let unbounded = select(&bounded); - assert_eq!( - unbounded.rejected[0].cause, - RejectionCause::LatencyOverRepeatInterval - ); - bounded.requirements.max_latency_ms = Some(f64::from(MINUTE)); - let selection = select(&bounded); - assert_eq!(selection.selected, candidates[1].id); - assert_eq!(selection.rejected[0].id, candidates[0].id); - assert_eq!( - selection.rejected[0].cause, - RejectionCause::LatencyBoundExceeded - ); - } - - /// When a maintained state's ingestion work needs more than one core, - /// its `k` workers' partial copies of the newest window are merged - /// before the query reads it: `(k − 1)` merges of one operation for - /// each of its states per window, on the query's chain. - #[test] - fn ingestion_work_over_one_core_adds_the_newest_windows_compaction() { - let latency = |rows_per_s: f64| { - let candidates = tumbling_window_candidates(rows_per_s); - price( - &candidates[1].dag, - &[repeating(MINUTE)], - &arriving(rows_per_s), - &CostInputs::default(), - ) - .unwrap() - .usage - .latency_per_query[0] - }; - assert_eq!(latency(10_000.0).compaction_ms, 0.0); - let fast = latency(1e10); - // 10,000 series, one state each per window, 1 ns per merge. - let merges = fast.compaction_ms / (10_000.0 * 1e-6); - assert!( - merges >= 1.0 && (merges - merges.round()).abs() < 1e-6, - "{fast:?}" - ); - } - - /// The newest window's compaction merges the `k` workers' partial - /// sketches with the state's measured query-time merge when the table - /// has one: each of the `(k − 1)·I` merges takes the measured 1,200 ns - /// instead of one operation. - #[test] - fn compaction_prices_merges_from_the_measured_merge() { - let candidates = tumbling_window_candidates(1e10); - let dag = &candidates[1].dag; - let kll = dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => Some(kind.params().clone()), - _ => None, - }) - .expect("a KLL build"); - let latency = price( - dag, - &[repeating(MINUTE)], - &arriving(1e10), - &measured(vec![sketch_entry(OperatorKind::SummaryMerge, kll, 1200.0)]), - ) - .unwrap() - .usage - .latency_per_query[0]; - assert!(latency.compaction_ms > 0.0, "{latency:?}"); - let merges = latency.compaction_ms / 1200e-6; - assert!( - merges >= 1.0 && (merges - merges.round()).abs() < 1e-6, - "{latency:?}" - ); - } - - // ── Cost models ────────────────────────────────────────────────────── - - /// [`CostInputs`] with `cost_model`. - fn under(cost_model: CostModel) -> CostInputs { - CostInputs { - cost_model, - ..CostInputs::default() - } - } - - /// The candidate's usage is what its nodes are charged for: priced at - /// one unit for a single resource under cost model 1 (CPU, scanned - /// bytes, memory byte-seconds or disk), the nodes' prices add up to the - /// candidate's usage of it. - #[test] - fn node_prices_add_up_to_the_candidates_usage() { - let only = |set: fn(&mut ResourceUnitCost)| { - let mut prices = ResourceUnitCost { - cost_per_cpu_op: 0.0, - cost_per_scanned_byte: 0.0, - cost_per_memory_byte_second: 0.0, - cost_per_provisioned_memory_byte_second: 0.0, - cost_per_provisioned_cpu_core_second: 0.0, - cost_per_disk_byte_second: 0.0, - }; - set(&mut prices); - CostInputs { - cost_model: CostModel::UsageOverTime(prices), - ..CostInputs::default() - } - }; - // Each resource's unit price and the candidate's usage of it. - type Resource = (fn(&mut ResourceUnitCost), fn(&ResourceUsage) -> f64); - let resources: [Resource; 4] = [ - (|p| p.cost_per_cpu_op = 1.0, |u| u.cpu_ops_per_second), - ( - |p| p.cost_per_scanned_byte = 1.0, - |u| u.scanned_bytes_per_second, - ), - ( - |p| p.cost_per_memory_byte_second = 1.0, - |u| u.average_memory_bytes, - ), - ( - |p| p.cost_per_disk_byte_second = 1.0, - |u| u.disk_bytes as f64, - ), - ]; - for candidate in tumbling_window_candidates(10_000.0) { - for (set, used) in resources { - let cost = price( - &candidate.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &only(set), - ) - .unwrap(); - let charged: f64 = cost - .per_node - .values() - .filter_map(|n| n.cost_per_second) - .sum(); - assert!( - close(charged, used(&cost.usage)), - "{}: {charged} vs {}", - candidate.id, - used(&cost.usage) - ); - assert_eq!(cost.usage.latency_per_query.len(), 1); - } - } - } - - /// Cost model 2 prices the candidate's peaks per second: provisioned - /// memory, cores and disk; its nodes carry no price. B1 (maintained - /// tumbling windows) needs a core for ingestion and one for the query; B2 (rebuilt - /// at query time) one. - #[test] - fn provisioned_capacity_prices_peak_memory_cores_and_disk() { - let prices = ResourceUnitCost { - cost_per_disk_byte_second: 1e-9, - ..ResourceUnitCost::illustrative() - }; - let settings = under(CostModel::ProvisionedCapacity(prices.clone())); - let candidates = tumbling_window_candidates(10_000.0); - let cores: Vec = candidates - .iter() - .map(|candidate| { - let cost = price( - &candidate.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &settings, - ) - .unwrap(); - let usage = &cost.usage; - let expected = prices.cost_per_provisioned_memory_byte_second - * usage.peak_memory_bytes as f64 - + prices.cost_per_provisioned_cpu_core_second * f64::from(usage.peak_cpu_cores) - + prices.cost_per_disk_byte_second * usage.disk_bytes as f64; - assert_eq!(cost.cost_per_second, Some(expected)); - assert!(cost.per_node.values().all(|n| n.cost_per_second.is_none())); - usage.peak_cpu_cores - }) - .collect(); - assert_eq!(cores, [1, 2]); - } - - /// Cost model 3 has no prices; a candidate over the cluster's cores is - /// invalid with the reason, and the rest are chosen by the objective. - #[test] - fn cluster_capacity_rejects_a_candidate_over_its_cores() { - let limits = |cpu_cores| ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores, - objective: ClusterObjective::LowestLatency, - }; - let candidates = tumbling_window_candidates(10_000.0); - let select = |cpu_cores| { - let settings = under(CostModel::ClusterCapacity(limits(cpu_cores))); - select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(&settings), - ) - .unwrap() - }; - let unlimited = select(None); - assert_eq!(unlimited.selected, candidates[1].id, "B1 merges faster"); - assert!(unlimited - .costs - .values() - .all(|c| c.cost_per_second.is_none())); - let one_core = select(Some(1)); - assert_eq!(one_core.selected, candidates[0].id); - let b1 = one_core - .rejected - .iter() - .find(|r| r.id == candidates[1].id) - .unwrap(); - assert_eq!(b1.cause, RejectionCause::OverClusterCapacity); - assert_eq!(b1.reason, "2 CPU cores at peak is over the cluster's 1"); - } - - /// Raw data kept on disk for query-time scans is counted once per - /// source, for the longest window read: two queries over `m` keep its - /// last 5 minutes, as the 5-minute query alone does. - #[test] - fn raw_data_on_disk_is_the_longest_window_per_source() { - let disk = |queries: &[&str]| { - let roots: Vec<_> = queries - .iter() - .map(|q| QueryRoot::Operator(lower_promql(q, AccuracyTarget::exact()))) - .collect(); - let dag = optimize_physical_asap("L", &roots, &[], &data(), false) - .unwrap() - .remove(0) - .dag; - priced(&dag, &recurrences(queries.len())).usage.disk_bytes - }; - let five_minutes = disk(&["sum_over_time(m[5m])"]); - assert!(five_minutes > disk(&["sum_over_time(m[1m])"])); - assert_eq!( - disk(&["sum_over_time(m[1m])", "sum_over_time(m[5m])"]), - five_minutes - ); - } - - /// Cost model 3's disk limit: B2 reads its 5-minute window of raw data - /// at query time and keeps it on disk; B1 keeps none. A cluster with - /// less disk than B2's raw data leaves only B1. - #[test] - fn cluster_capacity_rejects_a_candidate_over_its_disk() { - let candidates = tumbling_window_candidates(10_000.0); - let select = |disk_bytes| { - let settings = under(CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes, - cpu_cores: None, - objective: ClusterObjective::LeastMemory, - })); - select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(&settings), - ) - .unwrap() - }; - let unlimited = select(u64::MAX); - let raw = unlimited.costs[&candidates[0].id].usage.disk_bytes; - assert!(raw > 0); - assert_eq!(unlimited.costs[&candidates[1].id].usage.disk_bytes, 0); - assert_eq!(unlimited.selected, candidates[0].id, "B2 needs less memory"); - let small = select(raw - 1); - assert_eq!(small.selected, candidates[1].id); - let b2 = small - .rejected - .iter() - .find(|r| r.id == candidates[0].id) - .unwrap(); - assert_eq!(b2.cause, RejectionCause::OverClusterCapacity); - assert_eq!( - b2.reason, - format!("disk {raw} bytes is over the cluster's {} bytes", raw - 1) - ); - } - - /// State materialized on disk is counted as disk, not memory: the KLL - /// state kept on disk at ingestion time (its window being built and the - /// completed one) or for one evaluation at query time adds its bytes to - /// disk, nothing to average memory, and costs only its disk price. - #[test] - fn state_on_disk_is_counted_as_disk_not_memory() { - use asap_types::physical::Medium; - let settings = unit_costs(|c| c.cost_per_disk_byte_second = 1e-9); - let work_only = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let state = |dag: &PhysicalASAPDAG| node_of(dag, is_build); - for (on_disk, in_memory) in [ - (Materialization::MatAtIngestion(Medium::Disk), AT_INGESTION), - (Materialization::MatAtQuery(Medium::Disk), AT_QUERY), - ] { - let (disk_dag, memory_dag) = (kll_dag(on_disk), kll_dag(in_memory)); - let disk = price(&disk_dag, &recurrences(1), &data(), &settings).unwrap(); - let memory = price(&memory_dag, &recurrences(1), &data(), &settings).unwrap(); - // The bytes kept are the same, on disk instead of in memory. - let bytes = disk.usage.disk_bytes - memory.usage.disk_bytes; - assert!(bytes > 0); - assert!( - memory.per_node[&state(&memory_dag)] - .detail - .contains(&format!(" {bytes} bytes")), - "{in_memory:?}: {bytes}" - ); - assert!( - disk.usage.average_memory_bytes < memory.usage.average_memory_bytes, - "{on_disk:?}" - ); - assert_eq!( - disk.usage.peak_memory_bytes + bytes, - memory.usage.peak_memory_bytes, - "{on_disk:?}" - ); - // On disk, the state costs only its disk price beside the work. - let work = price(&memory_dag, &recurrences(1), &data(), &work_only) - .unwrap() - .per_node[&state(&memory_dag)] - .cost_per_second - .unwrap(); - let on_disk_node = &disk.per_node[&state(&disk_dag)]; - assert!( - close( - on_disk_node.cost_per_second.unwrap(), - work + 1e-9 * bytes as f64 - ), - "{on_disk_node:?}" - ); - } - } - - /// Under every cluster objective, two candidates with the same usage - /// tie, and the earlier is selected; the later is reported as tied. - /// Under the Pareto front neither dominates the other: both are on it. - #[test] - fn cluster_objectives_keep_the_earlier_of_tied_candidates() { - let b2 = tumbling_window_candidates(10_000.0).remove(0); - let twin = PhysicalWorkloadPlan { - id: format!("{}-twin", b2.id), - ..b2.clone() - }; - let candidates = [b2, twin]; - for objective in [ - ClusterObjective::LowestLatency, - ClusterObjective::LeastMemory, - ] { - let settings = under(CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective, - })); - let selection = select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(&settings), - ) - .unwrap(); - assert_eq!(selection.selected, candidates[0].id, "{objective:?}"); - let twin = &selection.rejected[0]; - assert_eq!(twin.id, candidates[1].id); - assert_eq!(twin.cause, RejectionCause::TiedLostOnEnumerationOrder); - assert!( - twin.reason.starts_with(&format!( - "tied with the selected candidate {}", - candidates[0].id - )), - "{objective:?}: {}", - twin.reason - ); - } - } - - /// Every Pass 2 window-composition candidate of a per-series p50 over - /// 2 min and a p99 over 5 min, both every minute, with both builds in - /// 1-min tumbling windows: `shared` picks one chain read by both (the 2-min query - /// merging its newest 2 tumbling windows) or a chain each. Its physical candidate - /// with every chain maintained at ingestion time. - fn nested_windows_at_ingestion(shared: bool, rows_per_s: f64) -> PhysicalWorkloadPlan { - let queries = [ - ("quantile_over_time(0.5, m[2m])", MINUTE), - ("quantile_over_time(0.99, m[5m])", MINUTE), - ]; - shared_windows_at_ingestion(queries, shared, rows_per_s) - } - - /// [`nested_windows_at_ingestion`] for two queries, each with its own - /// repeat interval in ms. - fn shared_windows_at_ingestion( - queries: [(&str, u32); 2], - shared: bool, - rows_per_s: f64, - ) -> PhysicalWorkloadPlan { - use asap_logical_asap_optimization::test_support as pcc; - use asap_types::stages::HowToComputeAggregateMeasure; - let inventory = pcc::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &queries - .iter() - .map(|(query, _)| QueryRoot::Operator(lower_promql(query, p99_target()))) - .collect::>(), - ), - ) - .unwrap(); - let kll: Vec = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap() - }) - .collect(); - let roots: Vec<_> = pcc::compose_logical_candidate(&inventory, &kll).unwrap(); - let rule_candidates = - asap_logical_asap_optimization::test_support::tumbling_window_candidates( - &roots, - &queries.map(|(_, interval_ms)| Some(u64::from(interval_ms))), - ); - let roots = - asap_logical_asap_optimization::test_support::window_composition_pass2_candidates( - &roots, - &rule_candidates, - ) - .unwrap() - .into_iter() - .find(|c| { - c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty() != shared - }) - .unwrap() - .roots - .unwrap(); - optimize_physical_asap( - "L1", - &roots, - &queries.map(|(_, interval_ms)| QueryWorkloadEntry { - predictability: asap_types::workload::Predictability::Predictable { - known_at: None, - }, - ..entry_with(p99_target(), repeating(interval_ms)) - }), - &arriving(rows_per_s), - false, - ) - .unwrap() - .into_iter() - .find(|c| { - c.dag.nodes.iter().all(|n| match n.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { - matches!(n.materialization, Materialization::MatAtIngestion(_)) - } - _ => n.materialization == Materialization::NotMaterialized, - }) - }) - .unwrap() - } - - /// One chain of 5 tumbling windows maintained at ingestion time serves a 5-min and - /// a 2-min window: the 2-min query's merge reads its newest 2 tumbling windows per - /// series, the 5-min query's all 5, and the plan costs less than a chain - /// per query, since each arriving row updates one tumbling window instead of two. - #[test] - fn a_merge_of_the_newest_tumbling_windows_is_priced_by_the_tumbling_windows_it_reads() { - let rows_per_s = 10_000.0; - let shared = nested_windows_at_ingestion(true, rows_per_s); - let chains: Vec<_> = shared - .dag - .nodes - .iter() - .filter(|n| is_chain(&n.payload)) - .collect(); - assert_eq!(chains.len(), 1); - let recurrence = [repeating(MINUTE), repeating(MINUTE)]; - let settings = CostInputs::default(); - let cost = price(&shared.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - let states_read = |newest: Option| -> u64 { - let merge = shared - .dag - .nodes - .iter() - .find(|n| { - matches!(&n.payload, Payload::ASAP(ASAPOp::SummaryMerge { newest_tumbling_windows, .. }) - if *newest_tumbling_windows == newest) - }) - .unwrap(); - let detail = &cost.per_node[&merge.id].detail; - detail - .strip_prefix("merge ") - .and_then(|rest| rest.split(' ').next()) - .and_then(|states| states.parse().ok()) - .unwrap_or_else(|| panic!("not a merge: {detail}")) - }; - let (newest_two, all_five) = (states_read(Some(2)), states_read(None)); - assert!(all_five > 0); - assert_eq!(newest_two * 5, all_five * 2); - let separate = nested_windows_at_ingestion(false, rows_per_s); - let separate_cost = - price(&separate.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - assert!( - cost.cost_per_second.unwrap() < separate_cost.cost_per_second.unwrap(), - "{:?} vs {:?}", - cost.cost_per_second, - separate_cost.cost_per_second - ); - } - - /// A 10-min quantile every minute (10 × 1-min tumbling windows) and one every 5 - /// min (2 × 5-min tumbling windows), maintained at ingestion time: one chain of - /// 1-min tumbling windows serves both, the 5-min query merging all 10 tumbling windows once - /// per 5 min, and costs less than a chain per query. - #[test] - fn queries_at_different_intervals_share_one_chain_at_ingestion_time() { - let rows_per_s = 10_000.0; - let queries = [ - ("quantile_over_time(0.5, m[10m])", MINUTE), - ("quantile_over_time(0.99, m[10m])", 5 * MINUTE), - ]; - let shared = shared_windows_at_ingestion(queries, true, rows_per_s); - let chains: Vec<_> = shared - .dag - .nodes - .iter() - .filter(|n| is_chain(&n.payload)) - .collect(); - assert_eq!(chains.len(), 1); - assert!(matches!( - chains[0].payload, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 10, - .. - }) - )); - let recurrence = [repeating(MINUTE), repeating(5 * MINUTE)]; - let settings = CostInputs::default(); - let cost = price(&shared.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - let separate = shared_windows_at_ingestion(queries, false, rows_per_s); - let separate_cost = - price(&separate.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - assert!( - cost.cost_per_second.unwrap() < separate_cost.cost_per_second.unwrap(), - "{:?} vs {:?}", - cost.cost_per_second, - separate_cost.cost_per_second - ); - assert!(cost.usage.average_memory_bytes < separate_cost.usage.average_memory_bytes); - // Each query merges all 10 tumbling windows per series at its own rate: once - // a minute and once per 5 min. - let mut merges: Vec<(u64, String)> = shared - .dag - .nodes - .iter() - .filter(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryMerge { .. }))) - .map(|n| { - let detail = &cost.per_node[&n.id].detail; - let states = detail - .strip_prefix("merge ") - .and_then(|rest| rest.split(' ').next()) - .and_then(|states| states.parse().ok()) - .unwrap_or_else(|| panic!("not a merge: {detail}")); - let rate = detail.split("; x ").nth(1).unwrap_or_default().to_string(); - (states, rate) - }) - .collect(); - merges.sort_by(|a, b| a.1.cmp(&b.1)); - assert_eq!(merges.len(), 2); - assert_eq!(merges[0].0, merges[1].0); - assert!(merges[0].1.starts_with("0.0033/s"), "{merges:?}"); - assert!(merges[1].1.starts_with("0.0167/s"), "{merges:?}"); - } - - /// A 1-hour quantile every minute (60 × 1-min tumbling windows) and every 45 s - /// (80 × 45-s tumbling windows) can share only a chain at their gcd, 240 × 15-s - /// tumbling windows. Maintained at ingestion time it keeps more states than both - /// per-query chains together, so it is priced higher than the - /// per-query chains maintained at ingestion time. - #[test] - fn a_shared_chain_at_a_fine_gcd_is_offered_but_costs_more() { - let rows_per_s = 10_000.0; - let queries = [ - ("quantile_over_time(0.5, m[1h])", MINUTE), - ("quantile_over_time(0.99, m[1h])", 45_000), - ]; - let shared = shared_windows_at_ingestion(queries, true, rows_per_s); - let chain = node_of(&shared.dag, is_chain); - let chain = shared.dag.nodes.iter().find(|n| n.id == chain).unwrap(); - assert!(matches!( - chain.payload, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 15_000, - tumbling_window_count: 240, - .. - }) - )); - let separate = shared_windows_at_ingestion(queries, false, rows_per_s); - let recurrence = [repeating(MINUTE), repeating(45_000)]; - let settings = CostInputs::default(); - let price_of = |candidate: &PhysicalWorkloadPlan| { - price( - &candidate.dag, - &recurrence, - &arriving(rows_per_s), - &settings, - ) - .unwrap() - .cost_per_second - .unwrap() - }; - assert!( - price_of(&shared) > price_of(&separate), - "{} vs {}", - price_of(&shared), - price_of(&separate) - ); - } - - /// With more than one ingest worker, each worker beyond the first holds - /// its own partial copy of a maintained state's newest window: B1's - /// peak memory grows by (k − 1) tumbling windows of states, its average memory - /// (and so cost model 1's price) does not. - #[test] - fn partial_copies_of_the_newest_window_add_to_peak_memory() { - let rows_per_s = 1e9; - let b1 = tumbling_window_candidates(rows_per_s).remove(1); - let chain = node_of(&b1.dag, is_chain); - let cost = price( - &b1.dag, - &[repeating(MINUTE)], - &arriving(rows_per_s), - &CostInputs::default(), - ) - .unwrap(); - let detail = &cost.per_node[&chain].detail; - let workers = cost.usage.peak_cpu_cores - 1; - assert!(workers > 1, "{workers}"); - // The chain keeps 6 tumbling windows (5 read and the one being built). - let kept: u64 = detail - .split("; keeps ") - .nth(1) - .and_then(|rest| rest.split(' ').next()) - .and_then(|bytes| bytes.parse().ok()) - .unwrap_or_else(|| panic!("{detail}")); - let copies = workers - 1; - let bytes = u64::from(copies) * (kept / 6); - assert!( - detail.contains(&format!( - "; {copies} partial copies of its newest window, {bytes} bytes" - )), - "{detail}" - ); - // At the peak, not on average: the kept bytes are B1's only - // averaged memory beside negligible query-time holds. - assert!(cost.usage.peak_memory_bytes >= kept + bytes); - assert!(cost.usage.average_memory_bytes < (kept + bytes) as f64); - } - - /// The KLL build of a `quantile_over_time` candidate, with its sketch - /// parameters, and the candidate. - fn kll_build() -> (PhysicalWorkloadPlan, PhysicalASAPNodeId, SketchParams) { - let (candidates, _) = per_series_kll(AccuracyScope::PerValue); - let candidate = candidates.into_iter().next().unwrap(); - let (build, sketch) = candidate - .dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some((n.id, kind.params().clone())), - _ => None, - }) - .unwrap(); - (candidate, build, sketch) - } - - /// An entry pricing `operator` at both phases on `sketch`. - fn sketch_entry( - operator: OperatorKind, - sketch: SketchParams, - ns_per_unit: f64, - ) -> OperatorCost { - OperatorCost { - operator, - phase: None, - sketch: Some(sketch), - exact: None, - measure: None, - ns_per_unit, - ns_per_invocation: 0.0, - memory_bytes_per_state: None, - } - } - - /// An entry pricing `operator` at `phase` with no family. - fn plain_entry( - operator: OperatorKind, - phase: Option, - ns_per_unit: f64, - ) -> OperatorCost { - OperatorCost { - operator, - phase, - sketch: None, - exact: None, - measure: None, - ns_per_unit, - ns_per_invocation: 0.0, - memory_bytes_per_state: None, - } - } - - /// Settings pricing from `operators`. - fn measured(operators: Vec) -> CostInputs { - CostInputs { - measured_operation_costs: MeasuredOperationCosts { - operators, - ..MeasuredOperationCosts::default() - }, - ..CostInputs::default() - } - } - - /// A measured update time for the candidate's KLL prices that build - /// from it (rows × measured ns, counted in operations of - /// `ms_per_cpu_op`) and reports `Measured`; every other node keeps its - /// formulas, reports `Analytical`, and has its CPU scaled by the - /// calibration factor, measured ÷ formula time of the build: 250 ns - /// against the formula's one 1-ns operation per row. - #[test] - fn a_measured_entry_prices_its_node_and_calibrates_the_others() { - let (candidate, build, sketch) = kll_build(); - // Scanning is free here, so each node's time is its CPU, which the - // factor scales (it leaves scanned bytes unscaled). - let cpu_time_only = |mut settings: CostInputs| { - settings.resource_speed.ms_per_scanned_byte = 0.0; - settings - }; - let before = price( - &candidate.dag, - &recurrences(1), - &data(), - &cpu_time_only(CostInputs::default()), - ) - .unwrap(); - let settings = cpu_time_only(measured(vec![sketch_entry( - OperatorKind::SummaryAgg, - sketch.clone(), - 250.0, - )])); - let after = price(&candidate.dag, &recurrences(1), &data(), &settings).unwrap(); - for (id, node) in &after.per_node { - let was = before.per_node[id].cpu_core_seconds_per_second; - if *id == build { - assert_eq!(node.price_source, NodePriceSource::Measured); - assert!(node.detail.contains("x 250.000 ns"), "{}", node.detail); - } else { - assert_eq!(node.price_source, NodePriceSource::Analytical, "node {id}"); - } - assert!( - close(node.cpu_core_seconds_per_second, 250.0 * was), - "node {id}: {} vs {was}", - node.cpu_core_seconds_per_second - ); - } - assert_eq!(before.source, COST_SOURCE); - assert_eq!(after.source, MIXED_COST_SOURCE); - // The candidate's CPU grows linearly in the measured time. - let cpu_at = |ns: f64| { - let settings = measured(vec![sketch_entry( - OperatorKind::SummaryAgg, - sketch.clone(), - ns, - )]); - price(&candidate.dag, &recurrences(1), &data(), &settings) - .unwrap() - .usage - .cpu_ops_per_second - }; - assert!(close( - (cpu_at(250.0) - cpu_at(100.0)) / (cpu_at(400.0) - cpu_at(100.0)), - 0.5 - )); - } - - /// A KLL with `k` = 469, as Pass 1 sizes it from the data, is priced - /// between the measured `k` = 200 and `k` = 1024, linearly in the log - /// of the state's size; a measured `k` matches exactly whatever failure - /// probability it was sized for, and a `k` beyond the measured ones - /// takes the nearest one's price. - #[test] - fn a_kll_between_two_measured_sizes_is_interpolated() { - let kll = |k, failure_probability| SketchParams::Kll { - k, - failure_probability, - }; - let table = MeasuredOperationCosts { - operators: vec![ - sketch_entry(OperatorKind::SummaryAgg, kll(200, None), 100.0), - sketch_entry(OperatorKind::SummaryAgg, kll(1024, None), 200.0), - ], - ..MeasuredOperationCosts::default() - }; - let size = |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; - let at = |k| { - lookup( - &table, - OperatorKind::SummaryAgg, - ExecutionTiming::IngestionTime, - StateFamily::Sketch(&kll(k, Some(0.001))), - size, - ) - .unwrap() - .ns_per_unit - }; - let expected = 100.0 - + 100.0 * (size(&kll(469, None)).ln() - size(&kll(200, None)).ln()) - / (size(&kll(1024, None)).ln() - size(&kll(200, None)).ln()); - assert!(close(at(469), expected), "{} vs {expected}", at(469)); - assert!(at(469) > 100.0 && at(469) < 200.0); - assert_eq!(at(200), 100.0); - assert_eq!(at(1024), 200.0); - assert_eq!(at(4096), 200.0); - assert_eq!(at(50), 100.0); - // Another sketch, operator or family has no entry. - assert!(lookup( - &table, - OperatorKind::SummaryMerge, - ExecutionTiming::QueryTime, - StateFamily::Sketch(&kll(469, None)), - size, - ) - .is_none()); - assert!(lookup( - &table, - OperatorKind::SummaryAgg, - ExecutionTiming::QueryTime, - StateFamily::Sketch(&SketchParams::DDSketch { alpha: 0.01 }), - size, - ) - .is_none()); - } - - /// An entry for the node's own phase wins over one for both phases. - #[test] - fn a_phase_entry_wins_over_one_for_both_phases() { - let table = MeasuredOperationCosts { - operators: vec![ - plain_entry(OperatorKind::Scan, None, 20.0), - plain_entry(OperatorKind::Scan, Some(ExecutionTiming::QueryTime), 150.0), - ], - ..MeasuredOperationCosts::default() - }; - let at = |phase| { - lookup(&table, OperatorKind::Scan, phase, StateFamily::None, |_| { - 1.0 - }) - .unwrap() - .ns_per_unit - }; - assert_eq!(at(ExecutionTiming::QueryTime), 150.0); - assert_eq!(at(ExecutionTiming::IngestionTime), 20.0); - } - - /// The physical candidates of a per-series p99 over 5 min, queried every - /// minute over data arriving at 10,000 rows per second: each Stage 1 - /// choice (the exact rebuild from raw data samples, KLL, DDSketch, ...) - /// with each Stage 2 materialization, ids `P-m`. - fn quantile_candidates() -> Vec { - let root = lower_promql("quantile_over_time(0.99, m[5m])", p99_target()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let mut out = Vec::new(); - for choice in 0..inventory.targets[0].choice_count() { - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, - &[choice], - ) - .unwrap(); - let roots = sized_as_pass1(&roots, &[p99_target()], &arriving(10_000.0)); - let candidates = - optimize_physical_asap("L", &roots, &[every_minute()], &arriving(10_000.0), false) - .unwrap(); - for (k, mut candidate) in candidates.into_iter().enumerate() { - candidate.id = format!("P{}-m{k}", choice + 1); - out.push(candidate); - } - } - out - } - - /// Whether `candidate` builds no summary: it rebuilds the quantile - /// exactly from raw data samples. - fn exact_rebuild(candidate: &PhysicalWorkloadPlan) -> bool { - !candidate.dag.nodes.iter().any(|n| { - matches!( - n.payload, - Payload::ASAP( - ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } - ) - ) - }) - } - - /// The exact rebuild of [`quantile_candidates`] and the KLL tumbling window - /// candidates ([`tumbling_window_candidates`]: B2 rebuilt at query time, B1 - /// maintained at ingestion time), with the KLL's parameters. - fn exact_and_kll() -> (Vec, SketchParams) { - let mut candidates: Vec<_> = quantile_candidates() - .into_iter() - .filter(exact_rebuild) - .collect(); - assert_eq!(candidates.len(), 1); - candidates[0].id = "EXACT".to_string(); - candidates.extend(tumbling_window_candidates(10_000.0)); - let kll = candidates[2] - .dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .expect("a KLL tumbling-window chain"); - (candidates, kll) - } - - /// Cost model 1 pricing CPU only (1 per ms of one core): the tests - /// below compare plans by the CPU their tables price. - fn cpu_only(operators: Vec) -> CostInputs { - CostInputs { - cost_model: CostModel::UsageOverTime(ResourceUnitCost { - cost_per_cpu_op: 1e-6, - cost_per_scanned_byte: 1e-7, - cost_per_memory_byte_second: 0.0, - cost_per_provisioned_memory_byte_second: 0.0, - cost_per_provisioned_cpu_core_second: 0.0, - cost_per_disk_byte_second: 0.0, - }), - ..measured(operators) - } - } - - /// The candidate Stage 3 selects among `candidates` for one p99 every - /// minute over 10,000 rows per second. - fn selected(candidates: &[PhysicalWorkloadPlan], settings: &CostInputs) -> WholePlanSelection { - select_among_plans( - candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(settings), - ) - .unwrap() - } - - /// The candidate cheapest when every node without an entry keeps its - /// formulas' price (no calibration), as Stage 3 priced before. - fn cheapest_uncalibrated(candidates: &[PhysicalWorkloadPlan], settings: &CostInputs) -> String { - let cost = |candidate: &PhysicalWorkloadPlan| { - plan_cost( - &candidate.dag, - &node_map(&candidate.dag), - &kept_tumbling_window_chains(&candidate.dag), - &[repeating(MINUTE)], - &[], - &arriving(10_000.0), - settings, - 1.0, - ) - .unwrap() - .0 - .cost_per_second - .unwrap() - }; - candidates - .iter() - .min_by(|a, b| cost(a).total_cmp(&cost(b))) - .unwrap() - .id - .clone() - } - - /// Calibration keeps the unmeasured nodes on the measured scale. A - /// table measuring the KLL as expensive (150 ns per row) and one exact - /// operator as cheap (the time range, 2 ns per row), but not the exact - /// plan's scan and aggregate, made the exact rebuild from raw data samples - /// look cheapest while the formulas kept its scan and aggregate at - /// their 1-ns scale; with the calibration factor the KLL maintained at - /// ingestion time (B1) is selected. With the full table, the exact - /// rebuild is selected when it really is cheaper (1 ns per scanned - /// sample and aggregated row), and B1 when raw reads and the exact - /// quantile cost 100 and 50 ns. - #[test] - fn calibration_keeps_unmeasured_nodes_on_the_measured_scale() { - let (candidates, kll) = exact_and_kll(); - let quantile = |ns_per_unit| OperatorCost { - measure: Some("quantile".to_string()), - ..plain_entry(OperatorKind::Aggregate, None, ns_per_unit) - }; - let kll_entries = || { - vec![ - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 150.0), - sketch_entry(OperatorKind::SummaryMerge, kll.clone(), 2_000.0), - sketch_entry(OperatorKind::SummaryEstimate, kll.clone(), 5_000.0), - plain_entry(OperatorKind::TimeRange, None, 2.0), - plain_entry(OperatorKind::Project, None, 1.0), - ] - }; - let partial = cpu_only(kll_entries()); - assert_eq!(cheapest_uncalibrated(&candidates, &partial), "EXACT"); - let selection = selected(&candidates, &partial); - assert_eq!(selection.selected, "P1-m1", "{:?}", selection.rejected); - let full = |scan_ns, aggregate_ns| { - let mut entries = kll_entries(); - entries.push(plain_entry(OperatorKind::Scan, None, scan_ns)); - entries.push(quantile(aggregate_ns)); - cpu_only(entries) - }; - assert_eq!(selected(&candidates, &full(1.0, 1.0)).selected, "EXACT"); - assert_eq!(selected(&candidates, &full(100.0, 50.0)).selected, "P1-m1"); - } - - /// The calibration factor is one heuristic, so a partial table can - /// still mis-rank plans: if the exact plan's scan and quantile really - /// cost 40 and 20 ns, the full table selects the exact rebuild, while - /// the partial table of - /// [`calibration_keeps_unmeasured_nodes_on_the_measured_scale`] prices - /// them at the KLL's ratio and selects B1, which then costs more than - /// the exact rebuild under the full table. - #[test] - fn a_partial_table_can_still_mis_rank_plans() { - let (candidates, kll) = exact_and_kll(); - let partial = vec![ - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 150.0), - sketch_entry(OperatorKind::SummaryMerge, kll.clone(), 2_000.0), - sketch_entry(OperatorKind::SummaryEstimate, kll.clone(), 5_000.0), - plain_entry(OperatorKind::TimeRange, None, 2.0), - plain_entry(OperatorKind::Project, None, 1.0), - ]; - let mut full = partial.clone(); - full.push(plain_entry(OperatorKind::Scan, None, 40.0)); - full.push(OperatorCost { - measure: Some("quantile".to_string()), - ..plain_entry(OperatorKind::Aggregate, None, 20.0) - }); - assert_eq!(selected(&candidates, &cpu_only(partial)).selected, "P1-m1"); - let full = selected(&candidates, &cpu_only(full)); - assert_eq!(full.selected, "EXACT"); - let cost = |id: &str| full.costs[id].cost_per_second.unwrap(); - assert!(cost("P1-m1") > cost("EXACT")); - } - - /// Reading raw data samples from the database holding them is the - /// deployment's cost. With every operator measured, the exact rebuild - /// from raw data samples at query time is the cheapest plan for a p99 every - /// minute while its raw reads are free (0 ns per sample fetched and - /// decoded at query time); priced at 200 ns per sample, it costs more - /// than the KLL maintained at ingestion time (B1), which is selected. - #[test] - fn external_raw_reads_make_the_query_time_raw_plan_costlier() { - let (candidates, kll) = exact_and_kll(); - let table = |raw_read_ns| { - cpu_only(vec![ - plain_entry( - OperatorKind::Scan, - Some(ExecutionTiming::IngestionTime), - 20.0, - ), - plain_entry( - OperatorKind::Scan, - Some(ExecutionTiming::QueryTime), - raw_read_ns, - ), - plain_entry(OperatorKind::TimeRange, None, 1.0), - plain_entry(OperatorKind::Project, None, 1.0), - OperatorCost { - measure: Some("quantile".to_string()), - ..plain_entry(OperatorKind::Aggregate, None, 10.0) - }, - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 30.0), - sketch_entry(OperatorKind::SummaryMerge, kll.clone(), 500.0), - sketch_entry(OperatorKind::SummaryEstimate, kll.clone(), 1_000.0), - ]) - }; - let free = selected(&candidates, &table(0.0)); - assert_eq!(free.selected, "EXACT", "{:?}", free.rejected); - let priced = selected(&candidates, &table(200.0)); - assert_eq!(priced.selected, "P1-m1", "{:?}", priced.rejected); - let cost = - |selection: &WholePlanSelection, id: &str| selection.costs[id].cost_per_second.unwrap(); - assert!(cost(&priced, "EXACT") > cost(&priced, "P1-m1")); - assert!(cost(&priced, "EXACT") > cost(&free, "EXACT")); - } - - /// A calibration factor below 1 (here a KLL build measured at 0.01 ns - /// per row against the formula's 1 ns) scales only the CPU of the - /// nodes without an entry, never their scanned bytes, so no node's CPU - /// goes negative: the query-time scan keeps its scanned bytes and - /// takes at least their scan time. - #[test] - fn a_calibration_factor_below_one_never_makes_cpu_negative() { - let (candidate, _, sketch) = kll_build(); - let before = price( - &candidate.dag, - &recurrences(1), - &data(), - &CostInputs::default(), - ) - .unwrap(); - let settings = measured(vec![sketch_entry(OperatorKind::SummaryAgg, sketch, 0.01)]); - let after = price(&candidate.dag, &recurrences(1), &data(), &settings).unwrap(); - assert!(after.usage.cpu_ops_per_second >= 0.0, "{:?}", after.usage); - for (id, node) in &after.per_node { - assert!( - node.cpu_core_seconds_per_second >= 0.0, - "node {id}: {}", - node.detail - ); - assert!(node.cost_per_second.unwrap() >= 0.0, "node {id}"); - } - assert_eq!( - after.usage.scanned_bytes_per_second, - before.usage.scanned_bytes_per_second - ); - } - - /// Each query request pays the table's `ns_per_evaluation` once: on - /// the node the query reads first, in its CPU and its latency. (Memory - /// is free here: the longer evaluation would hold its memory longer.) - #[test] - fn each_request_pays_the_per_evaluation_cost() { - let (candidate, _, _) = kll_build(); - let free_memory = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let before = price(&candidate.dag, &recurrences(1), &data(), &free_memory).unwrap(); - let settings = CostInputs { - measured_operation_costs: MeasuredOperationCosts { - ns_per_evaluation: 1e6, - ..MeasuredOperationCosts::default() - }, - ..free_memory - }; - let after = price(&candidate.dag, &recurrences(1), &data(), &settings).unwrap(); - // 1 ms per request, a request every 10 s. - assert!(close( - after.usage.latency_per_query[0].query_ms - before.usage.latency_per_query[0].query_ms, - 1.0 - )); - let core_seconds = |cost: &PlanCost| { - cost.per_node - .values() - .map(|n| n.cpu_core_seconds_per_second) - .sum::() - }; - assert!(close( - core_seconds(&after) - core_seconds(&before), - 1e-3 / 10.0 - )); - assert!(close( - after.cost_per_second.unwrap() - before.cost_per_second.unwrap(), - 0.1 - )); - } - - // A literal query has no operator to own its fee, but still incurs one - // request's CPU, latency and price; range steps do not multiply that fee. - #[test] - fn literal_requests_pay_overhead_without_operator_nodes() { - use asap_types::workload::{DemandEstimate, Rate}; - let roots = [QueryRoot::Scalar(ScalarExpr::Literal( - ScalarValue::Float64(1.0), - ))]; - let recurrence = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { - expected_rate: Rate(2000.0), - observed_at: None, - valid_for: None, - })); - let entries = [entry_with(AccuracyTarget::exact(), recurrence.clone())]; - let candidate = optimize_physical_asap("literal", &roots, &entries, &data(), false) - .unwrap() - .remove(0); - assert!(candidate.dag.nodes.is_empty()); - let mut settings = CostInputs::default(); - settings.measured_operation_costs.ns_per_evaluation = 2_000_000.0; - for range in [ - None, - Some(RangeEvaluation { - step_ms: 1000, - span_ms: 3000, - }), - ] { - let cost = self_calibrated_cost( - &candidate.dag, - &node_map(&candidate.dag), - &kept_tumbling_window_chains(&candidate.dag), - std::slice::from_ref(&recurrence), - &[range], - &data(), - &settings, - ) - .unwrap(); - let CostModel::UsageOverTime(prices) = &settings.cost_model else { - unreachable!() - }; - assert!(close( - cost.usage.cpu_ops_per_second, - 2.0 / settings.resource_speed.ms_per_cpu_op * 2000.0 - )); - assert!(close( - cost.cost_per_second.unwrap(), - prices.cost_per_cpu_op * cost.usage.cpu_ops_per_second - )); - assert!(cost.cost_per_second.unwrap() > 0.0); - assert_eq!(cost.usage.latency_per_query[0].query_ms, 2.0); - assert_eq!(cost.usage.peak_cpu_cores, 4); - assert!(cost.per_node.is_empty()); - assert_eq!(cost.source, MEASURED_COST_SOURCE); - } - // Cluster capacity must enforce overhead's throughput requirement too. - settings.cost_model = CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: Some(3), - objective: ClusterObjective::LowestLatency, - }); - let facts = data(); - let models = SelectionModels { - cost_inputs: &settings, - ..SelectionModels::builtin() - }; - let mut selector = PlanSelector::new(&entries, &facts, models, false).unwrap(); - selector.set_calibration(CalibrationFit::default()); - selector.offer(&candidate).unwrap(); - assert!(matches!( - selector.finish(), - Err(ErrorSelectingPlan::NoValidCandidate(_)) - )); - // Finite input coefficients can still overflow when multiplied by demand. - settings.measured_operation_costs.ns_per_evaluation = f64::MAX; - assert!(matches!( - price(&candidate.dag, &[recurrence], &facts, &settings), - Err((_, AnalyticalCostError::Overflow)) - )); - } - - /// A measured `memory_bytes_per_state` sizes the state a build keeps, - /// in place of the bytes the formulas derive from its parameters. - #[test] - fn measured_state_bytes_size_the_kept_state() { - let candidates = tumbling_window_candidates(10_000.0); - let maintained = &candidates[1]; - let kll = maintained - .dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .unwrap(); - let memory = |memory_bytes_per_state| { - let settings = measured(vec![OperatorCost { - memory_bytes_per_state, - ..sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 1.0) - }]); - price( - &maintained.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &settings, - ) - .unwrap() - .usage - .average_memory_bytes - }; - let formula = memory(None); - // The formula's state: each tumbling window sees its series' 60 samples (one - // minute of 10 000 rows per second over 10 000 series). - let samples = StateInputRows { - rows_per_state: Some(60), - non_negative: false, - }; - let half = memory(Some(sketch_shape(&kll, samples).1 as f64 / 2.0)); - assert!(half < formula && half > 0.0, "{half} vs {formula}"); - } - - // Oversized state configurations must be rejected as unrepresentable, - // never wrap to a cheap candidate or panic while pricing tumbling window choices. - #[test] - fn tumbling_window_state_byte_overflow_returns_a_cost_error() { - for candidate in tumbling_window_candidates(10_000.0) { - let kll = candidate - .dag - .nodes - .iter() - .find_map(|node| match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .unwrap(); - let settings = measured(vec![OperatorCost { - memory_bytes_per_state: Some(u64::MAX as f64), - ..sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll, 1.0) - }]); - assert!(matches!( - price( - &candidate.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &settings - ), - Err((_, AnalyticalCostError::Overflow)) - )); - } - } - - /// The reference pricer rejects a candidate it cannot price, as the - /// search does, rather than failing the whole selection. - #[test] - fn the_reference_pricer_rejects_an_unpriceable_candidate() { - let candidates = tumbling_window_candidates(10_000.0); - let kll = candidates[0] - .dag - .nodes - .iter() - .find_map(|node| match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .unwrap(); - let settings = measured(vec![OperatorCost { - memory_bytes_per_state: Some(u64::MAX as f64), - ..sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll, 1.0) - }]); - let entries = [entry_with(p99_target(), repeating(MINUTE))]; - let models = SelectionModels { - cost_inputs: &settings, - ..SelectionModels::builtin() - }; - let Err(ErrorSelectingPlan::NoValidCandidate(rejected)) = - select_among_plans(&candidates, &entries, &arriving(10_000.0), models) - else { - panic!("every candidate overflows, so none is valid"); - }; - assert_eq!(rejected.len(), candidates.len()); - assert!(rejected - .iter() - .all(|r| r.cause == RejectionCause::CostNotRepresentable)); - } - - /// The table of earlier versions, a JSON array of sketch operations, - /// still loads: each entry becomes its operator's entry at both phases, - /// an update both a `SummaryAgg` and a `SummaryTumblingWindowChain` entry. - #[test] - fn the_earlier_table_format_still_loads() { - let settings: CostInputs = serde_json::from_str( - r#"{"measured_operation_costs": [ - {"sketch": {"Kll": {"k": 200}}, "operation": "update", "cpu_ns_per_operation": 120.0}, - {"sketch": {"Kll": {"k": 200}}, "operation": "merge", "cpu_ns_per_operation": 900.0}]}"#, - ) - .unwrap(); - let kll = SketchParams::Kll { - k: 200, - failure_probability: None, - }; - assert_eq!( - settings.measured_operation_costs.operators, - vec![ - sketch_entry(OperatorKind::SummaryAgg, kll.clone(), 120.0), - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 120.0), - sketch_entry(OperatorKind::SummaryMerge, kll, 900.0), - ] - ); - let table: MeasuredOperationCosts = serde_json::from_str( - r#"{"operators": [{"operator": "Scan", "phase": "query_time", "ns_per_unit": 150.0}], - "ns_per_evaluation": 20000.0, - "provenance": {"executor_version": "0.1.0", "machine": "x86_64, 16 cores"}}"#, - ) - .unwrap(); - assert_eq!( - table.operators, - [plain_entry( - OperatorKind::Scan, - Some(ExecutionTiming::QueryTime), - 150.0 - )] - ); - let round_trip: MeasuredOperationCosts = - serde_json::from_str(&serde_json::to_string(&table).unwrap()).unwrap(); - assert_eq!(round_trip, table); - assert!(serde_json::from_str::( - r#"{"operators": [{"operator": "Scan", "ns": 1.0}]}"# - ) - .is_err()); - } - - /// With no entry for any node's operator and family (a CMS that no - /// node builds, a KLL merge in a plan without merges), every node is - /// priced from the formulas, as without a table. - #[test] - fn entries_for_other_operators_or_families_change_nothing() { - let (candidate, _, sketch) = kll_build(); - let before = price( - &candidate.dag, - &recurrences(1), - &data(), - &CostInputs::default(), - ) - .unwrap(); - let other = measured(vec![ - sketch_entry( - OperatorKind::SummaryAgg, - SketchParams::Cms { - width: 1024, - depth: 3, - }, - 50.0, - ), - sketch_entry(OperatorKind::SummaryMerge, sketch, 50.0), - ]); - let after = price(&candidate.dag, &recurrences(1), &data(), &other).unwrap(); - assert_eq!(after, before); - } - - /// Bad input: a negative or non-finite time, an entry naming two - /// families, two entries for one operator, phase and family, or a - /// table with `ms_per_cpu_op` 0. - #[test] - fn invalid_measured_tables_are_rejected() { - let cms = SketchParams::Cms { - width: 1024, - depth: 3, - }; - let entry = |ns| sketch_entry(OperatorKind::SummaryAgg, cms.clone(), ns); - let validate = |operators, ms_per_cpu_op| { - let mut settings = measured(operators); - settings.resource_speed.ms_per_cpu_op = ms_per_cpu_op; - settings.validate() - }; - for ns in [-1.0, f64::NAN] { - assert!(matches!( - validate(vec![entry(ns)], 1e-6), - Err(AnalyticalCostError::InvalidCalibration(..)) - )); - } - let duplicate = validate(vec![entry(1.0), entry(2.0)], 1e-6).unwrap_err(); - assert!( - matches!(duplicate, AnalyticalCostError::DuplicateOperatorCost(_)), - "{duplicate}" - ); - assert!( - duplicate - .to_string() - .contains("SummaryAgg at both phases on Cms"), - "{duplicate}" - ); - let two_families = OperatorCost { - exact: Some(asap_types::ir::schema::ExactKind::Sum), - ..entry(1.0) - }; - assert!(matches!( - validate(vec![two_families], 1e-6), - Err(AnalyticalCostError::AmbiguousOperatorCost(_)) - )); - assert_eq!( - validate(vec![entry(1.0)], 0.0), - Err(AnalyticalCostError::MeasuredCostsWithoutCpuSpeed(0.0)) - ); - let mut empty = CostInputs::default(); - empty.resource_speed.ms_per_cpu_op = 0.0; - assert!( - empty.validate().is_ok(), - "no table, no positive speed needed" - ); - } -} diff --git a/crates/plan-selection/src/milp/completion.rs b/crates/plan-selection/src/milp/completion.rs deleted file mode 100644 index 86bcc1f44..000000000 --- a/crates/plan-selection/src/milp/completion.rs +++ /dev/null @@ -1,306 +0,0 @@ -//! The cost of a plan given as values of the MILP's binaries: the cheapest -//! values of the continuous variables, found without a solver. -//! -//! Every continuous variable of the model is one of: -//! - a lower bound that minimization makes tight: `v ≥ f(others)` over one or -//! more rows (reach `a`, latency `L`, held time `G`/`H`/`W`, AND variables, -//! rate components, raw-retention levels, …). Its cheapest value is the -//! largest right-hand side, or 0; -//! - an equality `N = Σ a` (approximate values read); -//! - free below its upper rows only (`t`, reach through): its value is the -//! smallest upper bound, which leaves the most room to the rows it -//! relaxes. -//! -//! A row defines the continuous variable it bounds from below: the one with a -//! negative coefficient in `expression ≤ 0` (or any nonzero one in an -//! equality) that was created last, since the model creates each variable -//! after the variables its defining rows read. Rows of the families in -//! [`CHECK_ONLY_FAMILIES`] (upper bounds of reach, reader constraints, and -//! valid inequalities a plan's values meet anyway) define nothing. The -//! values are computed in creation order, repeated until no value changes, -//! then every row and bound is checked. A plan that fails a check is -//! infeasible as far as this completion goes (HiGHS's LP completion could -//! find another one; the warm start then just skips that plan). -//! -//! Each value is the smallest its defining rows allow, so at a plan's -//! binaries the values are the plan's own reach, latencies, held times and -//! rates, and the objective is what the model charges the plan: with the -//! exact cost terms of [`super::cost_terms`], what `asap_cost::plan_cost` -//! says (the warm start's test checks it). -use std::collections::HashMap; - -use good_lp::{IntoAffineExpression, Variable}; - -use super::model::SelectionMilp; - -/// Families whose rows only check values (upper bounds of reach, reader -/// constraints, rows tests add) and define no variable. -const CHECK_ONLY_FAMILIES: [&str; 4] = [ - "6 reach from above", - "9 reader constraints", - "V (slowest rate)", - "test", -]; - -/// The most passes over the variables before giving up on a fixpoint. -const MOST_PASSES: usize = 64; - -/// One row in `Σ coefficient · column + constant ≤ 0` (or `= 0`). -struct Row { - /// Range into `columns` and `coefficients`. - start: usize, - end: usize, - constant: f64, - equality: bool, -} - -/// The model's rows, compiled for repeated completions. -pub(crate) struct Completion { - column_of: HashMap, - /// The variables, by column. - variables: Vec, - binary: Vec, - max: Vec, - rows: Vec, - columns: Vec, - coefficients: Vec, - /// Per column, the rows defining it. - defining_rows: Vec>, - /// Per column with no defining row, the rows bounding it from above. - upper_rows: Vec>, - /// The columns with defining rows, in creation order. - defined: Vec, - /// The columns with neither (free continuous variables), in creation - /// order. - free: Vec, - objective: Vec<(usize, f64)>, - /// The family of each row. - families: Vec<&'static str>, -} - -/// A completed plan. -pub(crate) struct Completed { - /// Every variable's value, in creation order. - pub(crate) columns: Vec, - /// The objective (cost per second without the request fees). - pub(crate) cost_per_second: f64, -} - -impl Completion { - pub(crate) fn new(milp: &SelectionMilp<'_>) -> Self { - let builder = &milp.builder; - let mut column_of = HashMap::new(); - let mut variables = Vec::new(); - let mut binary = Vec::new(); - let mut max = Vec::new(); - for (column, (variable, definition)) in - builder.variables.iter_variables_with_def().enumerate() - { - column_of.insert(variable, column); - variables.push(variable); - binary.push(definition.is_integer()); - max.push(definition.get_max()); - } - let count = binary.len(); - let mut rows = Vec::with_capacity(builder.rows.len()); - let mut columns = Vec::new(); - let mut coefficients = Vec::new(); - let mut defining_rows: Vec> = vec![Vec::new(); count]; - let mut positive_rows: Vec> = vec![Vec::new(); count]; - for (index, (row, family)) in builder.rows.iter().zip(&builder.row_families).enumerate() { - let expression = row.expression(); - let start = columns.len(); - let mut defines: Option = None; - for (variable, coefficient) in expression.linear_coefficients() { - if coefficient == 0.0 { - continue; - } - let column = column_of[&variable]; - columns.push(column as u32); - coefficients.push(coefficient); - if binary[column] { - continue; - } - if coefficient > 0.0 { - positive_rows[column].push(index as u32); - } - if (coefficient < 0.0 || row.is_equality()) - && defines.is_none_or(|known| column > known) - { - defines = Some(column); - } - } - if !CHECK_ONLY_FAMILIES.contains(family) { - if let Some(column) = defines { - defining_rows[column].push(index as u32); - } - } - rows.push(Row { - start, - end: columns.len(), - constant: expression.constant(), - equality: row.is_equality(), - }); - } - let mut defined = Vec::new(); - let mut free = Vec::new(); - let mut upper_rows: Vec> = vec![Vec::new(); count]; - for column in 0..count { - if binary[column] { - continue; - } - if defining_rows[column].is_empty() { - upper_rows[column] = std::mem::take(&mut positive_rows[column]); - free.push(column); - } else { - defined.push(column); - } - } - let objective = milp - .objective_terms - .iter() - .map(|&(variable, coefficient)| (column_of[&variable], coefficient)) - .collect(); - Self { - column_of, - variables, - binary, - max, - rows, - columns, - coefficients, - defining_rows, - upper_rows, - defined, - free, - objective, - families: builder.row_families.clone(), - } - } - - /// The completion of a solver's columns (in creation order): its - /// binaries rounded, every continuous variable recomputed. - pub(crate) fn complete_columns(&self, columns: &[f64]) -> Result { - let binaries: Vec<(Variable, f64)> = self - .variables - .iter() - .zip(columns) - .zip(&self.binary) - .filter(|(_, &binary)| binary) - .map(|((&variable, &value), _)| (variable, if value > 0.5 { 1.0 } else { 0.0 })) - .collect(); - self.complete(&binaries) - } - - /// Every variable's value in `completed`. - pub(crate) fn values(&self, completed: &Completed) -> Vec<(Variable, f64)> { - self.variables - .iter() - .copied() - .zip(completed.columns.iter().copied()) - .collect() - } - - /// `Σ coefficient · value + constant` of row `row`, without column - /// `skip`, and the coefficient of `skip`. - fn row_value(&self, row: &Row, values: &[f64], skip: usize) -> (f64, f64) { - let mut sum = row.constant; - let mut own = 0.0; - for at in row.start..row.end { - let column = self.columns[at] as usize; - if column == skip { - own += self.coefficients[at]; - } else { - sum += self.coefficients[at] * values[column]; - } - } - (sum, own) - } - - /// The cheapest completion of `binaries` (every binary not listed is 0), - /// or why it has none here: the row or bound it breaks. - pub(crate) fn complete(&self, binaries: &[(Variable, f64)]) -> Result { - let mut values = vec![0.0; self.binary.len()]; - for &(variable, value) in binaries { - let column = self.column_of[&variable]; - if self.binary[column] { - values[column] = value; - } - } - // Defined columns: the largest lower bound, to a fixpoint. - let mut converged = false; - for _ in 0..MOST_PASSES { - let mut changed = false; - for &column in &self.defined { - let mut value: f64 = 0.0; - for &row in &self.defining_rows[column] { - let row = &self.rows[row as usize]; - let (rest, own) = self.row_value(row, &values, column); - // own · v + rest ≤ 0 (or = 0) with own < 0 for an inequality. - let bound = -rest / own; - if row.equality { - value = bound; - break; - } - value = value.max(bound); - } - if (value - values[column]).abs() > 1e-12 * value.abs().max(1.0) { - changed = true; - } - values[column] = value; - } - if !changed { - converged = true; - break; - } - } - if !converged { - return Err("no fixpoint of the lower bounds".into()); - } - // Free columns: the smallest upper bound. - for &column in &self.free { - let mut value = self.max[column]; - for &row in &self.upper_rows[column] { - let row = &self.rows[row as usize]; - let (rest, own) = self.row_value(row, &values, column); - value = value.min(-rest / own); - } - values[column] = if value.is_finite() { - value.max(0.0) - } else { - 0.0 - }; - } - // Every row and bound. - for (index, row) in self.rows.iter().enumerate() { - let mut sum = row.constant; - let mut magnitude = row.constant.abs(); - for at in row.start..row.end { - let term = self.coefficients[at] * values[self.columns[at] as usize]; - sum += term; - magnitude += term.abs(); - } - let tolerance = 1e-7 * magnitude.max(1.0); - if sum > tolerance || (row.equality && sum < -tolerance) { - return Err(format!( - "row {index} ({}) off by {sum:e}", - self.families[index] - )); - } - } - for (column, &value) in values.iter().enumerate() { - if value > self.max[column] + 1e-7 * self.max[column].abs().max(1.0) { - return Err(format!("column {column} at {value:e} above its bound")); - } - } - let cost_per_second = self - .objective - .iter() - .map(|&(column, coefficient)| coefficient * values[column]) - .sum(); - Ok(Completed { - columns: values, - cost_per_second, - }) - } -} diff --git a/crates/plan-selection/src/milp/cost_terms.rs b/crates/plan-selection/src/milp/cost_terms.rs deleted file mode 100644 index 7814f7b3c..000000000 --- a/crates/plan-selection/src/milp/cost_terms.rs +++ /dev/null @@ -1,1204 +0,0 @@ -//! #1095 §3: the cost model 1 objective of the group selection MILP. Every -//! number is `asap-cost`'s: a candidate's usage per execution and standing -//! cost ([`CandidateUsage`], from [`asap_cost::node_usage`] in -//! [`crate::groups`]), the schedule groups -//! ([`asap_cost::execution_demand::schedule_groups`]) and rates -//! ([`asap_cost::execution_demand::evaluation_rate`]) and the unit costs of -//! [`asap_cost::ResourceUnitCost`]. This module only turns them into -//! coefficients and the auxiliary rows that linearize them. -//! -//! A **run** is a candidate computed in an execution context (`p[ctx, c]`), -//! or a shareable candidate's own run (`x[m]`, keyed by its own context). -//! A run's usage is one or more [`RunTerm`]s, each an indicator and the -//! numbers that count when it is 1: -//! - one term on the run's own indicator when its usage is plan -//! independent; -//! - per producer choice, when its usage depends on which candidate of an -//! input's slot the context picks (a summary state's configuration, a -//! scan's window): `AND(run, p[ctx, c'], producer gives that output)`; -//! - for a query-time scan read at several windows in one context: one -//! term per set `S` of window kinds read, `δ_S`, the exact indicator that -//! exactly the kinds in `S` are read (from `σ_W`, the exact OR of the -//! reads at kind `W`), costed at the union of `S`'s windows -//! (`PhysicalASAPDAG::scan_window`). This needs no monotonicity: every -//! union is costed by `asap-cost` itself. -//! -//! **Executions per second** of a run: `rate(q)` in query context `q`; 1 in -//! each `I_r` and for an ingestion-time shareable candidate; for a query-time -//! shareable candidate `m` and the runs in its context, `Σ rate · z[m, G, -//! ℓ]` over its schedule groups and their levels, `z ≥ a[q, m]` for each -//! reader at that level or faster (a group's rate is its fastest reader's). -//! A context's runs other than `m`'s plain own term are summed once, `E[m] -//! ≥ Σ cost · indicator`, and charged per component, `V ≥ E[m] − Ē[m] (x[m] -//! − z)`, with `Σ rate · V ≥ (slowest rate) · E[m]`. -//! -//! **Held memory**: working memory of a run in query context `q` for `q`'s -//! latency at `q`'s rate (`H_query ≥ L[q] − L̄[q] (1 − indicator)`); `m`'s -//! `MatAtQuery(Memory)` result and the working memory of the runs in its -//! context until `m`'s slowest reader finishes (`H[m] ≥ L[q] − L̄[q] (1 − -//! a[q, m])`, then `W ≥ H[m] − H̄[m] (slack − Σ indicators)` per rate -//! component). -//! -//! **Raw retention**: per source, levels at each raw-data size a query-time -//! scan run may keep, `v[src, ℓ] ≥` each scan term keeping at least that -//! much; charged `Δbytes · unit disk cost`. -//! -//! **Pane retention**: a kept pane chain's usage at the fewest panes any -//! reader needs on `x[m]`, and each larger retention's increase on `π[m, -//! ℓ] ≥` the reach (as `plan_cost` counts it: through query-time runs) of -//! each query needing at least that many panes. -//! -//! Terms `plan_cost` defines differently from the above are counted in -//! [`SelectionMilp::inexact`] by reason. -use std::collections::{BTreeMap, BTreeSet, HashMap}; - -use asap_cost::execution_demand::{evaluation_rate, schedule_groups}; -use asap_cost::CostModel; -use asap_types::ir::physical_export::ScanWindow; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{NonASAPOp, Operator}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::stages::PhysicalCandidateId; -use asap_types::workload::bounds::source_name; -use asap_types::workload::{QueryRecurrence, RepeatedDemand}; -use good_lp::{constraint, Expression, IntoAffineExpression, Variable}; - -use crate::groups::{ - CandidateUsage, ContextKind, ResultSlot, RunKind, UsageAndOutput, UsageContext, -}; - -use super::model::{Reader, Scope, SelectionMilp, SelectionMilpError}; - -/// The numbers of one run `asap-cost` computed, as the objective and the -/// latency rows read them. -#[derive(Debug, Clone, Copy, Default, PartialEq)] -pub(crate) struct NodeNumbers { - /// Cost of one execution. - pub(crate) execution_cost: f64, - /// Cost per second of what it keeps whatever runs it. - pub(crate) standing_cost: f64, - /// What it adds to a reaching query's latency, ms. - pub(crate) latency_ms: f64, - /// Query-time memory held while an evaluation reading it runs. - pub(crate) held_bytes: f64, - /// Raw data a query-time scan keeps on disk. - pub(crate) raw_kept_bytes: f64, -} - -impl NodeNumbers { - pub(crate) fn of(usage: &CandidateUsage) -> Self { - Self { - execution_cost: usage.cost_per_execution.unwrap_or(0.0), - standing_cost: usage.standing_cost_per_second.unwrap_or(0.0), - latency_ms: usage.own_latency_ms, - held_bytes: usage.held_bytes as f64, - raw_kept_bytes: usage.raw_data_kept_bytes as f64, - } - } - - /// `self − other`, each number at least 0, and whether one was below. - fn minus(self, other: Self) -> (Self, bool) { - let fields = [ - self.execution_cost - other.execution_cost, - self.standing_cost - other.standing_cost, - self.latency_ms - other.latency_ms, - self.held_bytes - other.held_bytes, - self.raw_kept_bytes - other.raw_kept_bytes, - ]; - let negative = fields - .iter() - .any(|&field| field < -1e-12 * field.abs().max(1.0)); - let [execution_cost, standing_cost, latency_ms, held_bytes, raw_kept_bytes] = - fields.map(|field| field.max(0.0)); - ( - Self { - execution_cost, - standing_cost, - latency_ms, - held_bytes, - raw_kept_bytes, - }, - negative, - ) - } -} - -/// One term of a run: its numbers count, and it gives its readers output -/// `output`, when `indicator` is 1. At integral binaries exactly one term -/// of a used run is 1, except a kept pane chain's retention increases, -/// which add to its base term. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct RunTerm { - pub(crate) indicator: Expression, - pub(crate) numbers: NodeNumbers, - /// Whether `indicator` is the run's own variable (`p[ctx, c]` or - /// `x[m]`). - pub(crate) plain: bool, - /// Its output (an index below `CandidateData::output_count`). - pub(crate) output: usize, -} - -/// Run `(context, candidate)`'s terms (module documentation). Empty when -/// the run does not exist. -pub(crate) fn run_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - candidate: PhysicalCandidateId, -) -> Vec { - let Some(run) = milp.run_indicator(context, candidate) else { - return Vec::new(); - }; - let data = &milp.inputs.candidate_data[candidate.0]; - let Ok(base) = &data.usage else { - return Vec::new(); - }; - let base = *base; - for usage_context in data.usage_depends_on.clone() { - match usage_context { - UsageContext::ProducerOnACycle => milp.note_inexact("usage: a producer on a cycle"), - UsageContext::ProducerOutput - | UsageContext::ScanWindowOfReader - | UsageContext::PaneRetentionOfReachingQueries - | UsageContext::UncostableContext => {} - } - } - if milp.inputs.candidate_data[candidate.0].producer_usage_ambiguous { - milp.note_inexact("usage: a producer's output depends on its own producers"); - } - if !milp.inputs.candidate_data[candidate.0] - .usage_per_window - .is_empty() - { - if let Some(terms) = window_terms(milp, context, candidate, run) { - return terms; - } - } - if !milp.inputs.candidate_data[candidate.0] - .inputs_changing_usage - .is_empty() - { - return producer_terms(milp, context, candidate, run); - } - let mut terms = vec![RunTerm { - indicator: run.into(), - numbers: NodeNumbers::of(&base), - plain: true, - output: 0, - }]; - if !milp.inputs.candidate_data[candidate.0] - .usage_per_retained_panes - .is_empty() - { - pane_retention_terms(milp, context, candidate, &mut terms); - } - terms -} - -/// The indicator that, in `context`, the producer `producer` of a reader is -/// read and gives one of `outputs` (`None`: any output): `p[ctx, c']`, or -/// `AND(p[ctx, c'], t)` with `t ≥` the producer run's terms giving them. -/// `None` when the producer never gives them. -fn producer_gives( - milp: &mut SelectionMilp<'_>, - context: usize, - producer: PhysicalCandidateId, - outputs: Option<&BTreeSet>, -) -> Option { - let read = *milp.choice[context].get(&producer)?; - let Some(outputs) = outputs else { - return Some(read.into()); - }; - // The producer's run: computed here, its own run, or its run in an - // `I_r`. - let producer_run = match milp.kind(producer) { - _ if milp.computes(context, producer) => context, - RunKind::Shareable => milp.context_of(producer)?, - RunKind::Ingestion => milp.ingestion_context_read_by(context)?, - RunKind::PerContext => return None, - }; - if milp.terms_in_progress.contains(&(producer_run, producer)) { - milp.note_inexact("usage: a producer's output on a cycle of links (every output kept)"); - return Some(read.into()); - } - let producer_terms = milp.run_terms(producer_run, producer); - if producer_terms - .iter() - .all(|term| outputs.contains(&term.output)) - { - return Some(read.into()); - } - let gives: Vec<&RunTerm> = producer_terms - .iter() - .filter(|term| outputs.contains(&term.output)) - .collect(); - if gives.is_empty() { - return None; - } - let key = ( - producer_run, - producer, - outputs.iter().copied().collect::>(), - ); - let gives = match milp.producer_gives.get(&key) { - Some(&gives) => gives, - None => { - let sum: Expression = gives.iter().map(|term| term.indicator.clone()).sum(); - let variable = milp - .builder - .continuous("t_outputs (a producer gives outputs)", Some(1.0)); - milp.builder.row( - "t_outputs (a producer gives outputs)", - constraint!(variable >= sum), - ); - milp.producer_gives.insert(key, variable); - variable - } - }; - if milp.computes(context, producer) { - return Some(gives.into()); - } - Some( - milp.builder - .and_at_least("t_producers (AND)", &[read.into(), gives.into()]) - .into(), - ) -} - -/// A producer choice with the outputs it stands for (`None`: any output). -type OutputsOfProducer = (PhysicalCandidateId, Option>); - -/// Terms of a run whose usage or output depends on its producers (and -/// their outputs): one per choice, on `AND(run, producer gives)` over the -/// changing inputs. -fn producer_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - candidate: PhysicalCandidateId, - run: Variable, -) -> Vec { - milp.terms_in_progress.insert((context, candidate)); - let data = &milp.inputs.candidate_data[candidate.0]; - let changing = data.inputs_changing_usage.clone(); - let base = *data.usage.as_ref().expect("costed"); - let same = |known: &Option, gives: &Option| match (known, gives) - { - (Some(known), Some(gives)) => known == gives, - (None, None) => true, - _ => false, - }; - let mut table: Vec<(Vec, Option)> = Vec::new(); - for (choices, gives) in &data.usage_per_producers { - let choices: Vec = choices - .iter() - .map(|choice| { - ( - choice.class, - choice.output.map(|output| BTreeSet::from([output])), - ) - }) - .collect(); - if changing.len() == 1 { - if let Some((known, _)) = table - .iter_mut() - .find(|(known, known_gives)| known[0].0 == choices[0].0 && same(known_gives, gives)) - { - if let (Some(outputs), Some(more)) = (&mut known[0].1, &choices[0].1) { - outputs.extend(more); - } - continue; - } - } - table.push((choices, *gives)); - } - let producers_here: Vec> = changing - .iter() - .map(|&input| { - milp.inputs.producer_candidates_per_input[candidate.0][input] - .iter() - .copied() - .filter(|producer| milp.choice[context].contains_key(producer)) - .filter(|producer| milp.slot(*producer) != milp.slot(candidate)) - .collect() - }) - .collect(); - let mut covered: Vec> = vec![BTreeSet::new(); changing.len()]; - let mut terms = Vec::new(); - for (choices, gives) in &table { - let mut per_input: Vec = Vec::with_capacity(choices.len()); - for (position, (producer, outputs)) in choices.iter().enumerate() { - if !producers_here[position].contains(producer) { - break; - } - covered[position].insert(*producer); - match producer_gives(milp, context, *producer, outputs.as_ref()) { - Some(indicator) => per_input.push(indicator), - None => break, - } - } - if per_input.len() < choices.len() { - continue; - } - let mut all: Vec = vec![run.into()]; - all.extend(per_input); - // A multi-slot input reads the choice's slot only through its link. - for (position, (producer, _)) in choices.iter().enumerate() { - if let Some(links) = milp - .slot_links - .get(&(context, candidate, changing[position])) - { - let slot = milp.slot(*producer); - if let Some(&(_, link)) = links.iter().find(|(at, _)| *at == slot) { - all.push(link.into()); - } - } - } - let count = all.len() as f64; - match gives { - None => { - let sum: Expression = all.into_iter().sum(); - milp.builder - .row("t_producers (AND)", constraint!(sum <= count - 1.0)); - } - Some((usage, output)) => { - let indicator = milp.builder.and_at_least("t_producers (AND)", &all); - terms.push(RunTerm { - indicator: indicator.into(), - numbers: NodeNumbers::of(usage), - plain: false, - output: *output, - }); - } - } - } - // A producer missing from the table: its reads carry the first costed - // usage. - let mut missing = false; - for (position, producers) in producers_here.iter().enumerate() { - for &producer in producers { - if covered[position].contains(&producer) { - continue; - } - missing = true; - let read = milp.choice[context][&producer]; - let indicator = milp - .builder - .and_at_least("t_producers (AND)", &[run.into(), read.into()]); - terms.push(RunTerm { - indicator: indicator.into(), - numbers: NodeNumbers::of(&base), - plain: false, - output: 0, - }); - } - } - if missing { - milp.note_inexact("usage: a producer missing from its usage table"); - } - if terms.is_empty() { - milp.builder - .row("t_producers (AND)", constraint!(run <= 0.0)); - } - milp.terms_in_progress.remove(&(context, candidate)); - terms -} - -/// The union of `windows`, as [`PhysicalASAPDAG::scan_window`] computes it -/// for one scan read at each: the nearest end and the farthest start. -pub(crate) fn window_union(windows: impl IntoIterator) -> ScanWindow { - let mut near = i64::MAX; - let mut far = i64::MIN; - for window in windows { - match window { - ScanWindow::Unbounded => return ScanWindow::Unbounded, - ScanWindow::Within { - end_offset_ms, - width_ms, - } => { - near = near.min(end_offset_ms); - far = far.max(end_offset_ms.saturating_add(width_ms)); - } - } - } - if near > far { - return ScanWindow::Unbounded; - } - ScanWindow::Within { - end_offset_ms: near, - width_ms: far.saturating_sub(near), - } -} - -/// The window a scan run read at `kinds` scans, as `asap_cost::plan_cost` -/// prices it: their union ([`window_union`]) when every kind is bounded; -/// with an unbounded read (a reader that is not a time range, a pane chain -/// or a time shift, or a read as an output), the trailing window of the -/// widest bounded kind (`plan_cost`'s `widest_window_read_ms`), or -/// `Unbounded` when there is none. -pub(crate) fn scan_window_read_at( - kinds: impl IntoIterator + Clone, -) -> ScanWindow { - let unbounded = kinds - .clone() - .into_iter() - .any(|kind| kind == ScanWindow::Unbounded); - if !unbounded { - return window_union(kinds); - } - let widest = kinds - .into_iter() - .filter_map(|kind| match kind { - ScanWindow::Within { width_ms, .. } => Some(width_ms), - ScanWindow::Unbounded => None, - }) - .max(); - match widest { - Some(width_ms) => ScanWindow::Within { - end_offset_ms: 0, - width_ms, - }, - None => ScanWindow::Unbounded, - } -} - -/// The windows reader `reader` reads scan `scan` at (one per path through -/// time shifts); `Unbounded` when unknown. -fn reader_windows( - milp: &SelectionMilp<'_>, - scan: PhysicalCandidateId, - reader: PhysicalCandidateId, -) -> Vec { - milp.inputs.candidate_data[scan.0] - .windows_per_reader - .iter() - .find(|(known, _)| *known == reader) - .map(|(_, read)| read.clone()) - .unwrap_or_else(|| vec![ScanWindow::Unbounded]) -} - -/// Terms of query-time scan run `(context, scan)` whose usage depends on -/// the window its readers read (module documentation). `None` when its -/// readers read one window kind and it has no other dependence handled -/// here: the caller then uses its other terms. -fn window_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - scan: PhysicalCandidateId, - run: Variable, -) -> Option> { - if let Some(cached) = milp.window_cache.get(&(context, scan)) { - return cached.clone(); - } - // Each read: (reading context, window kind, reader's indicator). - let mut reads: Vec<(usize, ScanWindow, Expression)> = Vec::new(); - let reading_contexts: Vec = if milp.kind(scan) == RunKind::Shareable { - (0..milp.choice.len()) - .filter(|&other| milp.choice[other].contains_key(&scan)) - .collect() - } else { - vec![context] - }; - for &reading in &reading_contexts { - for reader in milp.readers[reading] - .get(&scan) - .cloned() - .unwrap_or_default() - { - let (window, indicator) = match reader { - Reader::Candidate(reader, _) => { - let windows = reader_windows(milp, scan, reader); - if windows.len() > 1 { - milp.note_inexact("a scan read at several windows by one reader (union)"); - } - ( - window_union(windows), - Expression::from(milp.choice[reading][&reader]), - ) - } - Reader::Output(_) => (ScanWindow::Unbounded, Expression::from(1.0)), - Reader::OwnInput(_) => { - let ContextKind::Shared(own) = milp.inputs.contexts[reading].kind else { - continue; - }; - // `m` itself reads the scan: at its own window. - ( - window_union(reader_windows(milp, scan, own)), - Expression::from(milp.selected[&own]), - ) - } - }; - reads.push((reading, window, indicator)); - } - } - let mut kinds: Vec = Vec::new(); - for (_, window, _) in &reads { - if !kinds.contains(window) { - kinds.push(*window); - } - } - let usage_at = |milp: &SelectionMilp<'_>, window: ScanWindow| { - let found = milp.inputs.candidate_data[scan.0] - .usage_per_window - .iter() - .find(|(known, _, _)| *known == window) - .map(|&(_, usage, output)| (NodeNumbers::of(&usage), output)); - if found.is_none() { - super::warm_start::debug(|| { - format!( - "scan {} has no usage at {window:?}; costed at {:?}", - scan.0, - milp.inputs.candidate_data[scan.0] - .usage_per_window - .iter() - .map(|(w, _, _)| *w) - .collect::>() - ) - }); - } - found - }; - let terms = match kinds.len() { - 0 => None, - 1 => match usage_at(milp, scan_window_read_at([kinds[0]])) { - Some((numbers, output)) => Some(vec![RunTerm { - indicator: run.into(), - numbers, - plain: true, - output, - }]), - None => { - // `asap-cost` cannot cost the scan so: no plan reads it so. - milp.builder - .row("δ (scan window sets)", constraint!(run <= 0.0)); - Some(Vec::new()) - } - }, - count => { - // `σ_W`: the exact OR, over the reading contexts, of "the - // context reads the scan and a reader at kind `W` is used" - // (`w`, exact from both sides at integral binaries). - let mut sigma = Vec::with_capacity(count); - for kind in &kinds { - let mut per_context: Vec = Vec::new(); - for &reading in &reading_contexts { - let at_kind: Vec<&Expression> = reads - .iter() - .filter(|(at, window, _)| *at == reading && window == kind) - .map(|(_, _, indicator)| indicator) - .collect(); - if at_kind.is_empty() { - continue; - } - let read_here = milp.choice[reading][&scan]; - let used = milp.builder.continuous("σ (scan window kinds)", Some(1.0)); - let mut upper = Expression::from(0.0); - for indicator in at_kind { - let indicator = indicator.clone(); - upper += indicator.clone(); - milp.builder.row( - "σ (scan window kinds)", - constraint!(used >= indicator + read_here - 1.0), - ); - } - milp.builder - .row("σ (scan window kinds)", constraint!(used <= read_here)); - milp.builder - .row("σ (scan window kinds)", constraint!(used <= upper)); - per_context.push(used); - } - let or = match per_context.as_slice() { - [one] => *one, - many => { - let or = milp.builder.continuous("σ (scan window kinds)", Some(1.0)); - let mut upper = Expression::from(0.0); - for &used in many { - milp.builder - .row("σ (scan window kinds)", constraint!(or >= used)); - upper += used; - } - milp.builder - .row("σ (scan window kinds)", constraint!(or <= upper)); - or - } - }; - sigma.push(or); - } - let mut terms = Vec::new(); - assert!( - count < usize::BITS as usize, - "a scan run with {count} window kinds has too many unions to encode" - ); - for set in 1usize..(1 << count) { - let window = scan_window_read_at( - (0..count) - .filter(|bit| set & (1 << bit) != 0) - .map(|bit| kinds[bit]), - ); - let members = set.count_ones() as f64; - let mut sum = Expression::from(0.0); - for (bit, &or) in sigma.iter().enumerate() { - if set & (1 << bit) != 0 { - sum += or; - } else { - sum -= or; - } - } - let Some((numbers, output)) = usage_at(milp, window) else { - // `asap-cost` cannot cost the scan at this union: no - // plan reads exactly these kinds. - milp.builder - .row("δ (scan window sets)", constraint!(sum <= members - 1.0)); - continue; - }; - let exactly = milp.builder.continuous("δ (scan window sets)", Some(1.0)); - milp.builder.row( - "δ (scan window sets)", - constraint!(exactly >= sum - (members - 1.0)), - ); - terms.push(RunTerm { - indicator: exactly.into(), - numbers, - plain: false, - output, - }); - } - Some(terms) - } - }; - milp.window_cache.insert((context, scan), terms.clone()); - terms -} - -/// For a shareable kept pane chain with usage depending on the panes it -/// retains: its base term at the fewest panes, plus one term per larger -/// retention, `π[m, ℓ] ≥` the reach of each query needing at least `ℓ`. -fn pane_retention_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - chain: PhysicalCandidateId, - terms: &mut Vec, -) { - let data = &milp.inputs.candidate_data[chain.0]; - let mut levels = data.usage_per_retained_panes.clone(); - levels.sort_by_key(|(panes, _)| *panes); - let per_query = data.retained_panes_per_query.clone(); - let Some(&(_, base)) = levels.first() else { - return; - }; - terms[0].numbers = NodeNumbers::of(&base); - let at_query_time = milp.runs_at_query_time(context); - // Per query, the indicator that it reaches the chain as `plan_cost` - // counts reach (through query-time runs). - let mut reached: Vec<(usize, Vec)> = Vec::new(); - for &(query, _) in &per_query { - let mut indicators = Vec::new(); - if let Some(&reach) = milp.reach.get(&(query, chain)) { - if at_query_time - || milp.physical.candidates[chain.0].timing == ExecutionTiming::QueryTime - { - indicators.push(Expression::from(reach)); - reached.push((query, indicators)); - continue; - } - } - if let Some(&direct) = milp.choice[query].get(&chain) { - indicators.push(direct.into()); - } - for &reader in milp - .shared_readers - .get(&chain) - .cloned() - .unwrap_or_default() - .iter() - { - let reader_context = milp.inputs.context_of_shareable[&reader]; - if !milp.runs_at_query_time(reader_context) { - continue; - } - if let Some(&above) = milp.reach.get(&(query, reader)) { - let read = milp.choice[reader_context][&chain]; - let and = milp - .builder - .and_at_least("π (pane retention levels)", &[above.into(), read.into()]); - indicators.push(and.into()); - } - } - reached.push((query, indicators)); - } - let mut below = NodeNumbers::of(&base); - for &(panes, usage) in &levels[1..] { - let numbers = NodeNumbers::of(&usage); - let (increase, negative) = numbers.minus(below); - below = numbers; - if negative { - milp.note_inexact("a longer pane retention with a smaller number"); - } - if increase.execution_cost > 0.0 && at_query_time { - milp.note_inexact("pane retention changing a query-time execution cost"); - } - if increase.latency_ms > 0.0 || increase.held_bytes > 0.0 { - milp.note_inexact("pane retention changing latency or held memory (not encoded)"); - } - let level = milp - .builder - .continuous("π (pane retention levels)", Some(1.0)); - for (query, indicators) in &reached { - let needs = per_query - .iter() - .any(|&(q, retained)| q == *query && retained >= panes); - if !needs { - continue; - } - for indicator in indicators { - let indicator = indicator.clone(); - milp.builder - .row("π (pane retention levels)", constraint!(level >= indicator)); - } - } - terms.push(RunTerm { - indicator: level.into(), - numbers: NodeNumbers { - latency_ms: 0.0, - held_bytes: 0.0, - ..increase - }, - plain: false, - output: terms[0].output, - }); - } -} - -/// Each ingestion-time shareable candidate's compaction term for its -/// readers' latency, ms, where `plan_cost` charges one: `(k − 1) · states · -/// merge time`, `k` the cores its ingestion work takes. With at most one -/// second of ingestion work per second below it, `k = 1` and the term is 0; -/// a larger subtree is not encoded (counted as inexact). -pub(crate) fn compaction_ms(milp: &mut SelectionMilp<'_>) -> HashMap { - let inputs = milp.inputs; - let mut compaction = HashMap::new(); - for &shareable in inputs.context_of_shareable.keys() { - if milp.physical.candidates[shareable.0].timing != ExecutionTiming::IngestionTime { - continue; - } - let mut seen = BTreeSet::new(); - let mut stack = vec![shareable]; - let mut most_work_ms = 0.0; - while let Some(class) = stack.pop() { - if !seen.insert(class) { - continue; - } - if let Ok(usage) = &inputs.candidate_data[class.0].usage { - most_work_ms += usage.work_ms_per_execution; - } - stack.extend( - inputs.producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied(), - ); - } - if most_work_ms > 1_000.0 { - milp.note_inexact("compaction with more than one ingest worker (not encoded)"); - } - compaction.insert(shareable, 0.0); - } - compaction -} - -/// A schedule group's readers with their own rates, and its rate levels. -type GroupLevels = (Vec<(usize, f64)>, Vec); - -/// How the runs of a context execute per second. -#[derive(Debug, Clone)] -enum ContextRate { - /// A fixed rate (a query's, 1 at ingestion time, or a query-time - /// shareable candidate whose possible readers run it at one rate). - Fixed(f64), - /// `Σ rate · z` over a query-time shareable candidate's schedule-group - /// levels. - Components(Vec<(Variable, f64)>), -} - -/// Each context's rate (module documentation), and the queries' own -/// rates. -fn context_rates( - milp: &mut SelectionMilp<'_>, -) -> Result<(Vec, Vec), SelectionMilpError> { - let horizon = milp.cost_inputs.non_repeating_query_horizon_seconds; - let cost = |error: asap_cost::analytical::AnalyticalCostError| { - SelectionMilpError::Cost(error.to_string()) - }; - let query_rate: Vec = milp - .entries - .iter() - .map(|entry| evaluation_rate([(&entry.recurrence, entry.range_evaluation)], horizon)) - .collect::>() - .map_err(cost)?; - let mut rates = Vec::with_capacity(milp.inputs.contexts.len()); - milp.builder.scope = Scope::SharedContexts; - for context in 0..milp.inputs.contexts.len() { - let rate = match milp.inputs.contexts[context].kind { - ContextKind::Query(query) => ContextRate::Fixed(query_rate[query]), - ContextKind::Ingestion(_) => ContextRate::Fixed(1.0), - ContextKind::Shared(m) => { - let per_request = match milp.physical.candidates[m.0].materialization { - Materialization::MatAtIngestion(_) | Materialization::NotMaterialized => { - rates.push(ContextRate::Fixed(1.0)); - continue; - } - Materialization::MatAtQuery(_) => false, - Materialization::MatAtQueryKeptAcrossEvaluations(_) => true, - }; - let readers = milp.queries_reaching(m); - let demand = |query: usize| { - let entry = &milp.entries[query]; - ( - &entry.recurrence, - if per_request { - None - } else { - entry.range_evaluation - }, - ) - }; - if unknown_phase_joins_several( - readers.iter().map(|&query| &milp.entries[query].recurrence), - ) { - milp.note_inexact("schedule: an unknown phase among several known ones"); - } - let groups = schedule_groups(readers.iter().map(|&query| demand(query)), horizon) - .map_err(cost)?; - let mut levels_per_group: Vec = Vec::new(); - for group in groups { - let members: Vec = group - .readers - .iter() - .map(|&position| readers[position]) - .collect(); - let alone: Vec = members - .iter() - .map(|&query| evaluation_rate([demand(query)], horizon)) - .collect::>() - .map_err(cost)?; - let fastest = alone.iter().copied().fold(0.0, f64::max); - let full = group.evaluations_per_second; - let mut levels: Vec = alone.clone(); - if (full - fastest).abs() > 1e-9 * full.abs().max(1e-12) { - milp.note_inexact("schedule: a group's rate is not its fastest reader's"); - levels = vec![full]; - } - levels.sort_by(f64::total_cmp); - levels.dedup(); - levels_per_group.push((members.into_iter().zip(alone).collect(), levels)); - } - if let [(_, levels)] = levels_per_group.as_slice() { - if let [level] = levels.as_slice() { - rates.push(ContextRate::Fixed(*level)); - continue; - } - } - let mut components = Vec::new(); - for (members, levels) in levels_per_group { - let mut below = 0.0; - for &level in &levels { - let z = milp.builder.continuous("z (schedule groups)", Some(1.0)); - for &(query, own) in &members { - if own >= level || levels.len() == 1 { - let reach = milp.reach[&(query, m)]; - milp.builder - .row("z (schedule groups)", constraint!(z >= reach)); - } - } - components.push((z, level - below)); - below = level; - } - } - ContextRate::Components(components) - } - }; - rates.push(rate); - } - milp.builder.scope = Scope::Global; - Ok((rates, query_rate)) -} - -/// Whether some interval has readers of unknown phase and readers of two -/// or more known phases. -fn unknown_phase_joins_several<'r>(recurrences: impl Iterator) -> bool { - let mut phases: BTreeMap)> = BTreeMap::new(); - for recurrence in recurrences { - match recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) => { - phases.entry(interval.0).or_default().0 = true; - } - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval, - evaluation_phase, - }) => { - phases - .entry(interval.0) - .or_default() - .1 - .insert(evaluation_phase.0 % u64::from(interval.0.max(1))); - } - _ => {} - } - } - phases - .values() - .any(|(unknown, known)| *unknown && known.len() > 1) -} - -/// The objective (cost model 1, module documentation) and its rows. -pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), SelectionMilpError> { - let CostModel::UsageOverTime(unit_cost) = milp.cost_inputs.cost_model.clone() else { - return Err(SelectionMilpError::UnsupportedCostModel("not cost model 1")); - }; - let horizon = milp.cost_inputs.non_repeating_query_horizon_seconds; - let cost = |error: asap_cost::analytical::AnalyticalCostError| { - SelectionMilpError::Cost(error.to_string()) - }; - let request_ms = milp.cost_inputs.measured_operation_costs.ns_per_evaluation * 1e-6; - if request_ms > 0.0 { - for entry in milp.entries { - let requests = evaluation_rate([(&entry.recurrence, None)], horizon).map_err(cost)?; - milp.request_cost_per_second += unit_cost.cost_per_cpu_op * request_ms - / milp.cost_inputs.resource_speed.ms_per_cpu_op - * requests; - } - } - let (rates, query_rate) = context_rates(milp)?; - let memory = unit_cost.cost_per_memory_byte_second; - let mut objective: Vec<(Variable, f64)> = Vec::new(); - let mut charge = |expression: &Expression, coefficient: f64| { - if coefficient != 0.0 { - for (variable, factor) in expression.linear_coefficients() { - objective.push((variable, factor * coefficient)); - } - } - }; - // Per query-time shareable context: the execution cost of its runs - // other than `m`'s plain own term, and per slot (`None`: `m`'s own run) - // the most it can be. - let mut context_execution: BTreeMap, f64>)> = - BTreeMap::new(); - // `H[m]` per query-time shareable context. - let mut held_time: HashMap = HashMap::new(); - let runs = milp.runs(); - for (context, candidate) in runs { - let terms = milp.run_terms(context, candidate); - milp.builder.scope = milp.scope_of(context); - let candidate_data = &milp.physical.candidates[candidate.0]; - let (timing, materialization) = (candidate_data.timing, candidate_data.materialization); - let own = milp.inputs.contexts[context].kind == ContextKind::Shared(candidate); - // The most this run adds to its context's execution cost: all its - // terms at 1. - let mut run_execution = 0.0; - for term in &terms { - let numbers = term.numbers; - charge(&term.indicator, numbers.standing_cost); - match &rates[context] { - ContextRate::Fixed(rate) => charge(&term.indicator, numbers.execution_cost * rate), - ContextRate::Components(components) => { - if numbers.execution_cost == 0.0 { - } else if term.plain && own { - for &(component, rate) in components { - charge(&component.into(), numbers.execution_cost * rate); - } - } else { - let (sum, _) = context_execution - .entry(context) - .or_insert_with(|| (Expression::from(0.0), BTreeMap::new())); - *sum += numbers.execution_cost * term.indicator.clone(); - run_execution += numbers.execution_cost; - } - } - } - // Held memory. - if numbers.held_bytes <= 0.0 || timing != ExecutionTiming::QueryTime { - continue; - } - // Per byte held per latency unit. - let per_byte_ms = memory * numbers.held_bytes / 1_000.0 * milp.ms_per_latency_unit; - match milp.inputs.contexts[context].kind { - ContextKind::Query(query) => { - let latency = milp.latency[query]; - let most = milp.latency_max[query]; - let held = milp - .builder - .continuous("H_query (held for a query's latency)", None); - let indicator = term.indicator.clone(); - milp.builder.row( - "H_query (held for a query's latency)", - constraint!(held >= latency - most * (1.0 - indicator)), - ); - charge(&held.into(), per_byte_ms * query_rate[query]); - } - ContextKind::Shared(m) => { - if own && materialization != Materialization::MatAtQuery(Medium::Memory) { - milp.note_inexact("held bytes of a shareable candidate not in memory"); - } - let (longest, most_held) = *held_time.entry(context).or_insert_with(|| { - let longest = milp - .builder - .continuous("H (held until m's slowest reader)", None); - let mut most_held: f64 = 0.0; - for query in 0..milp.latency.len() { - if let Some(&reach) = milp.reach.get(&(query, m)) { - let latency = milp.latency[query]; - let most = milp.latency_max[query]; - most_held = most_held.max(most); - milp.builder.row( - "H (held until m's slowest reader)", - constraint!(longest >= latency - most * (1.0 - reach)), - ); - } - } - (longest, most_held) - }); - let components: Vec<(Option, f64)> = match &rates[context] { - ContextRate::Fixed(rate) => vec![(None, *rate)], - ContextRate::Components(components) => components - .iter() - .map(|&(component, rate)| (Some(component), rate)) - .collect(), - }; - for (component, rate) in components { - let mut both = Expression::from(0.0); - let mut slack = 0.0; - if let Some(component) = component { - both += component; - slack += 1.0; - } - if !(term.plain && own) { - both += term.indicator.clone(); - slack += 1.0; - } - if slack == 0.0 { - charge(&longest.into(), per_byte_ms * rate); - continue; - } - let held = milp - .builder - .continuous("W (held time × schedule group)", None); - milp.builder.row( - "W (held time × schedule group)", - constraint!(held >= longest - most_held * (slack - both)), - ); - charge(&held.into(), per_byte_ms * rate); - } - } - ContextKind::Ingestion(_) => {} - } - } - if let Some((_, most_per_slot)) = context_execution.get_mut(&context) { - let slot = (!own).then(|| milp.inputs.slot[candidate.0]); - let entry = most_per_slot.entry(slot).or_default(); - *entry = entry.max(run_execution); - } - } - // A context's execution at its varying rate: `V ≥ E − Ē (x[m] − z)` per - // component `z`, `Ē` the most `E` can be: per slot, its costliest run. - milp.builder.scope = Scope::SharedContexts; - for (context, (sum, most_per_slot)) in context_execution { - let most: f64 = if super::model::tests_loose_big_m() { - // Before: every term's cost. - sum.clone() - .linear_coefficients() - .map(|(_, cost)| cost) - .sum() - } else { - most_per_slot.values().sum() - }; - let ContextRate::Components(components) = &rates[context] else { - continue; - }; - let ContextKind::Shared(m) = milp.inputs.contexts[context].kind else { - continue; - }; - let region = milp - .builder - .continuous("V (context execution at a rate component)", None); - milp.builder.row( - "V (context execution at a rate component)", - constraint!(region >= sum), - ); - let active = milp.selected[&m]; - let slowest = components - .iter() - .map(|&(_, rate)| rate) - .fold(f64::INFINITY, f64::min); - let mut at_rates = Expression::from(0.0); - for &(component, rate) in components { - let at_rate = milp - .builder - .continuous("V (context execution at a rate component)", None); - milp.builder.row( - "V (context execution at a rate component)", - constraint!(at_rate >= region - most * (active - component)), - ); - charge(&at_rate.into(), rate); - at_rates += rate * at_rate; - } - if slowest.is_finite() && slowest > 0.0 { - milp.builder.row( - "V (slowest rate)", - constraint!(at_rates >= slowest * region), - ); - } - } - milp.builder.scope = Scope::Global; - raw_retention(milp, unit_cost.cost_per_disk_byte_second, &mut charge); - milp.objective_terms = objective; - Ok(()) -} - -/// Raw retention: per source, the largest raw data a query-time scan run of -/// the plan keeps, by levels. -fn raw_retention( - milp: &mut SelectionMilp<'_>, - disk: f64, - charge: &mut impl FnMut(&Expression, f64), -) { - let mut kept: BTreeMap> = BTreeMap::new(); - for (context, candidate) in milp.runs() { - let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = - &milp.physical.physical_node(candidate).operator - else { - continue; - }; - if milp.physical.candidates[candidate.0].timing != ExecutionTiming::QueryTime { - continue; - } - let source = source_name(source).to_string(); - for term in milp.run_terms(context, candidate) { - let bytes = term.numbers.raw_kept_bytes; - if bytes > 0.0 { - kept.entry(source.clone()) - .or_default() - .push((bytes, term.indicator)); - } - } - } - for (_, scans) in kept { - let mut levels: Vec = scans.iter().map(|(bytes, _)| *bytes).collect(); - levels.sort_by(f64::total_cmp); - levels.dedup(); - let mut below = 0.0; - for level in levels { - let at_least = milp - .builder - .continuous("v (raw retention levels)", Some(1.0)); - for (bytes, indicator) in &scans { - if *bytes >= level { - let indicator = indicator.clone(); - milp.builder.row( - "v (raw retention levels)", - constraint!(at_least >= indicator), - ); - } - } - charge(&at_least.into(), disk * (level - below)); - below = level; - } - } -} diff --git a/crates/plan-selection/src/milp/decode.rs b/crates/plan-selection/src/milp/decode.rs deleted file mode 100644 index 5b8b53466..000000000 --- a/crates/plan-selection/src/milp/decode.rs +++ /dev/null @@ -1,162 +0,0 @@ -//! #1095 Step 6 (§2.0, "Decode and check"): the group selection MILP's raw -//! solution as a [`DagOfSelectedCandidates`], one node per candidate run. -//! -//! - A candidate computed in a context (`p[ctx, c] = 1`, a `NotMaterialized` -//! query-time candidate in a query's or a shareable candidate's context, -//! or a `NotMaterialized` ingestion-time candidate in an `I_r`) is one node per -//! context: selected in two contexts, it is two nodes; read by two -//! readers in one context, it is one. -//! - A selected shareable candidate (`x[m] = 1`) is one node, whoever reads -//! it. -//! - In each context, input `n` of a computed node (or of `m`'s own node in -//! `m`'s context) reads the context's selected candidate among its -//! producers: the node computed there, the shareable candidate's node, or -//! the candidate's node in `I_r(m)` (read in ingestion-time `m`'s context). -//! - Output reference `r` is the selected output candidate in `r`'s query -//! context. -//! -//! Only the nodes the outputs reach are kept. The MILP never selects an -//! unread candidate (row 4), so this drops nothing at a solution whose -//! objective equals `plan_cost`. - -use std::collections::{BTreeMap, BTreeSet}; - -use asap_types::stages::{ - output_references, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateId, - SelectedEdge, SelectedNodeId, -}; - -use crate::groups::{MilpInputs, ResultSlot, RunKind}; - -/// One run of a candidate in the solution. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -enum Run { - /// Candidate `.1` computed in context `.0`. - Computed(usize, PhysicalCandidateId), - /// Shareable candidate `.0`. - Shared(PhysicalCandidateId), -} - -/// The selected candidates of a solution: per context, `p[ctx, c] = 1`, and -/// the shareable candidates with `x[m] = 1`. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub(crate) struct ChosenCandidates { - pub(crate) chosen: Vec>, - pub(crate) selected: BTreeSet, - /// Per multi-slot input `(context, reader, input)`, the slot it reads. - pub(crate) slot_reads: BTreeMap<(usize, PhysicalCandidateId, usize), ResultSlot>, -} - -/// The selection as a DAG of candidate runs over `physical` (module -/// documentation), or why it is not one (an output or an input with no -/// selected producer). The caller validates it -/// ([`DagOfSelectedCandidates::validate`]). -pub(crate) fn decode( - physical: &PhysicalASAPMemoCandidates, - inputs: &MilpInputs, - chosen: &ChosenCandidates, -) -> Result { - // The node a read of `producer` in `context` resolves to. - let run_of = |context: usize, producer: PhysicalCandidateId| -> Option { - if !chosen.chosen[context].contains(&producer) { - return None; - } - let kind = inputs.kind[producer.0]; - if inputs.contexts[context].computes(kind) { - return Some(Run::Computed(context, producer)); - } - match kind { - RunKind::Shareable => chosen - .selected - .contains(&producer) - .then_some(Run::Shared(producer)), - RunKind::Ingestion => inputs - .ingestion_context_read_by(context) - .filter(|&ingestion| chosen.chosen[ingestion].contains(&producer)) - .map(|ingestion| Run::Computed(ingestion, producer)), - RunKind::PerContext => None, - } - }; - // The run's context for its inputs, and its candidate. - let inputs_of = |run: Run| -> Result, String> { - let (context, candidate) = match run { - Run::Computed(context, candidate) => (context, candidate), - Run::Shared(candidate) => ( - *inputs - .context_of_shareable - .get(&candidate) - .ok_or_else(|| format!("shareable candidate {} has no context", candidate.0))?, - candidate, - ), - }; - let mut reads = Vec::new(); - for (input, producers) in inputs.producer_candidates_per_input[candidate.0] - .iter() - .enumerate() - { - let slot_read = match run { - Run::Computed(..) => chosen.slot_reads.get(&(context, candidate, input)), - Run::Shared(_) => None, - }; - let producer = producers - .iter() - .filter(|producer| { - matches!(run, Run::Shared(_)) - || inputs.slot[producer.0] != inputs.slot[candidate.0] - }) - .filter(|producer| slot_read.is_none_or(|&slot| inputs.slot[producer.0] == slot)) - .find_map(|&producer| run_of(context, producer)) - .ok_or_else(|| { - format!( - "input {input} of candidate {} in context {context} has no selected producer", - candidate.0 - ) - })?; - reads.push((input, producer)); - } - Ok(reads) - }; - let outputs: Vec = output_references(&physical.query_roots) - .into_iter() - .enumerate() - .map(|(reference, (query, _))| { - inputs.output_candidates[reference] - .iter() - .find_map(|&candidate| run_of(query, candidate)) - .ok_or_else(|| format!("output reference {reference} has no selected candidate")) - }) - .collect::>()?; - // Number the runs the outputs reach, depth first from the outputs. - let mut node_of: BTreeMap = BTreeMap::new(); - let mut reads_of: BTreeMap> = BTreeMap::new(); - let mut candidate_of_node = Vec::new(); - let mut stack: Vec = outputs.iter().rev().copied().collect(); - while let Some(run) = stack.pop() { - if node_of.contains_key(&run) { - continue; - } - node_of.insert(run, SelectedNodeId(candidate_of_node.len())); - candidate_of_node.push(match run { - Run::Computed(_, candidate) | Run::Shared(candidate) => candidate, - }); - let reads = inputs_of(run)?; - stack.extend(reads.iter().map(|&(_, producer)| producer)); - reads_of.insert(run, reads); - } - let mut edges = Vec::new(); - for (run, &consumer) in &node_of { - for &(input, producer) in &reads_of[run] { - edges.push(SelectedEdge { - producer: node_of[&producer], - consumer, - input, - }); - } - } - edges.sort_by_key(|edge| (edge.consumer.0, edge.input)); - Ok(DagOfSelectedCandidates { - candidate_of_node, - edges, - node_of_output_reference: outputs.iter().map(|run| node_of[run]).collect(), - }) -} diff --git a/crates/plan-selection/src/milp/lp_diagnosis.rs b/crates/plan-selection/src/milp/lp_diagnosis.rs deleted file mode 100644 index d8bdc113f..000000000 --- a/crates/plan-selection/src/milp/lp_diagnosis.rs +++ /dev/null @@ -1,302 +0,0 @@ -//! #1095 §6.1: where the selection MILP's LP relaxation loses its bound -//! (measurement only, behind the `lp-diagnosis` feature). With -//! `ASAP_STAGE3_LP_DIAGNOSIS=`, each solve appends one JSON line: -//! - `lp`: the LP relaxation of the model as built, and `root`: HiGHS's -//! bound after its root node (presolve, cuts), both from the warm start; -//! - `lp_without`: the LP with one row family dropped, per family: how much -//! of the LP bound that family's rows hold up; -//! - `lp_tightened`: the LP with valid rows added: `Σ terms ≥ run` for each -//! run charged only on its configuration or window terms (at integral -//! binaries one of them is 1); -//! - `objective_by_family`: the objective split by the family of each -//! charged variable, at the LP's solution, at the warm start and at the -//! incumbent, so `incumbent − lp` splits into one share per family; -//! - `runs`: the charged `p` and `x` variables with the largest difference -//! between the incumbent and the LP, with their values in both. -//! -//! A diagnosed run's wall time is not comparable with an undiagnosed one. -//! The LPs and the root node run without a time limit: an LP solve and a -//! one-node MIP solve end on their own. -use std::collections::BTreeMap; -use std::time::Instant; - -use good_lp::solvers::highs::highs; -use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, SolverModel}; - -use super::model::{objective_scale, SelectionMilp}; - -/// The environment variable naming the file the diagnosis appends to. -const DIAGNOSIS_ENV: &str = "ASAP_STAGE3_LP_DIAGNOSIS"; - -/// The runs listed per solve. -const RUNS_SHOWN: usize = 40; - -/// One LP (or root node) solved. -#[derive(Debug, Clone, serde::Serialize)] -struct Bound { - /// The bound, cost per second (`None` when HiGHS gave none). - per_second: Option, - status: String, - ms: u64, -} - -/// The LP solves, before the MILP's. -pub(crate) struct Diagnosis { - path: String, - lp: Bound, - root: Bound, - lp_without: BTreeMap<&'static str, Bound>, - lp_tightened: Bound, - tightening_rows: usize, - /// The LP's columns. - lp_columns: Vec, - /// The objective, by column. - objective: Vec<(usize, f64)>, - column_families: Vec<&'static str>, - /// Per `p` and `x` column, what it stands for. - labels: BTreeMap, -} - -impl Diagnosis { - /// The LP solves of `milp` from `start` (columns), when the diagnosis is - /// asked for. - pub(crate) fn run(milp: &SelectionMilp<'_>, start: Option<&[f64]>) -> Option { - let path = std::env::var(DIAGNOSIS_ENV).ok()?; - let definitions: Vec<_> = milp - .builder - .variables - .iter_variables_with_def() - .map(|(variable, definition)| (variable, definition.clone())) - .collect(); - let column_of: std::collections::HashMap<_, usize> = definitions - .iter() - .enumerate() - .map(|(column, (variable, _))| (*variable, column)) - .collect(); - let objective: Vec<(usize, f64)> = milp - .objective_terms - .iter() - .map(|&(variable, coefficient)| (column_of[&variable], coefficient)) - .collect(); - let start_values: Option> = start.map(|start| { - definitions - .iter() - .map(|(variable, _)| *variable) - .zip(start.iter().copied()) - .collect() - }); - let scale = objective_scale(&milp.objective_terms, start_values.as_deref()); - // The relaxation: the same variables (same indices), none integral. - let relaxed = || { - let mut variables = ProblemVariables::new(); - for (_, definition) in &definitions { - variables.add( - variable() - .min(definition.get_min()) - .max(definition.get_max()), - ); - } - variables - }; - let objective_expression: Expression = milp - .objective_terms - .iter() - .map(|&(variable, coefficient)| coefficient * scale * variable) - .sum(); - let solve = |variables: ProblemVariables, rows: Vec, root: bool| { - let started = Instant::now(); - let problem = variables - .minimise(objective_expression.clone()) - .using(highs) - .with_all(rows); - let mut model = problem.try_into_inner().expect("a HiGHS model"); - model.set_option("output_flag", false); - model.set_option("threads", milp.highs_threads); - if root { - // The root node, then stop: its bound is the root bound. - model.set_option("mip_max_nodes", 1); - model.set_option("mip_rel_gap", 0.0); - model.set_option("mip_abs_gap", 0.0); - if let Some(start) = start { - let _ = model.try_set_solution(Some(start), None, None, None); - } - } - let Ok(solved) = model.try_solve() else { - return ( - Bound { - per_second: None, - status: "solver error".into(), - ms: started.elapsed().as_millis() as u64, - }, - Vec::new(), - ); - }; - let value = if root { - solved.double_info_value(c"mip_dual_bound").ok() - } else if solved.status() == highs::HighsModelStatus::Optimal { - Some(solved.objective_value()) - } else { - None - }; - let columns = if root { - Vec::new() - } else { - solved.get_solution().columns().to_vec() - }; - ( - Bound { - per_second: value.filter(|v| v.is_finite()).map(|v| v / scale), - status: format!("{:?}", solved.status()), - ms: started.elapsed().as_millis() as u64, - }, - columns, - ) - }; - let rows = &milp.builder.rows; - let families = &milp.builder.row_families; - let (lp, lp_columns) = solve(relaxed(), rows.clone(), false); - let (root, _) = solve(milp.builder.variables.clone(), rows.clone(), true); - let mut lp_without = BTreeMap::new(); - let present: std::collections::BTreeSet<&'static str> = families.iter().copied().collect(); - for family in present { - let kept: Vec = rows - .iter() - .zip(families) - .filter(|(_, &f)| f != family) - .map(|(row, _)| row.clone()) - .collect(); - lp_without.insert(family, solve(relaxed(), kept, false).0); - } - let tightening = cover_rows(milp); - let tightening_rows = tightening.len(); - let mut tightened = rows.clone(); - tightened.extend(tightening); - let (lp_tightened, _) = solve(relaxed(), tightened, false); - let candidate = |c: asap_types::stages::PhysicalCandidateId| { - let data = &milp.physical.candidates[c.0]; - format!( - "c{} {} {:?} {:?}", - c.0, - milp.physical.physical_node(c).operator.kind_name(), - data.timing, - data.materialization - ) - }; - let mut labels = BTreeMap::new(); - for (context, choices) in milp.choice.iter().enumerate() { - let place = match milp.inputs.contexts[context].kind { - crate::groups::ContextKind::Query(query) => format!("q{query}"), - crate::groups::ContextKind::Ingestion(pipeline) => match pipeline { - crate::groups::IngestionPipeline::Window(window) => { - format!("I[{}ms@{}]", window.width_ms, window.phase_ms) - } - crate::groups::IngestionPipeline::Alone(m) => format!("I[alone m{}]", m.0), - }, - crate::groups::ContextKind::Shared(m) => format!("m{}", m.0), - }; - for (&c, variable) in choices { - labels.insert(column_of[variable], format!("p[{place}] {}", candidate(c))); - } - } - for (&m, variable) in &milp.selected { - labels.insert(column_of[variable], format!("x {}", candidate(m))); - } - Some(Self { - labels, - path, - lp, - root, - lp_without, - lp_tightened, - tightening_rows, - lp_columns, - objective, - column_families: milp.builder.column_families.clone(), - }) - } - - /// Appends the line, with the warm start's and the incumbent's columns - /// (completed) and the MILP's results. - pub(crate) fn write( - self, - start: Option<&[f64]>, - incumbent: Option<&[f64]>, - objective_per_second: f64, - dual_bound_per_second: f64, - ) { - let by_family = |columns: Option<&[f64]>| -> Option> { - let columns = columns.filter(|columns| !columns.is_empty())?; - let mut split: BTreeMap<&'static str, f64> = BTreeMap::new(); - for &(column, coefficient) in &self.objective { - *split.entry(self.column_families[column]).or_default() += - coefficient * columns[column]; - } - Some(split) - }; - // Per labeled column: (LP charge, incumbent charge, LP value, - // incumbent value). - let mut runs: Vec<(String, f64, f64, f64, f64)> = Vec::new(); - if let Some(incumbent) = incumbent.filter(|columns| !columns.is_empty()) { - let mut charge: BTreeMap = BTreeMap::new(); - for &(column, coefficient) in &self.objective { - if self.labels.contains_key(&column) { - *charge.entry(column).or_default() += coefficient; - } - } - for (column, coefficient) in charge { - let (lp, at) = (self.lp_columns[column], incumbent[column]); - runs.push(( - self.labels[&column].clone(), - coefficient * lp, - coefficient * at, - lp, - at, - )); - } - runs.sort_by(|a, b| (b.2 - b.1).abs().total_cmp(&(a.2 - a.1).abs())); - runs.truncate(RUNS_SHOWN); - } - let line = serde_json::json!({ - "runs": runs, - "variables": self.column_families.len(), - "objective_per_second": objective_per_second, - "dual_bound_per_second": dual_bound_per_second, - "lp": self.lp, - "root": self.root, - "lp_without": self.lp_without, - "lp_tightened": self.lp_tightened, - "tightening_rows": self.tightening_rows, - "objective_by_family": { - "lp": by_family(Some(&self.lp_columns)), - "warm_start": by_family(start), - "incumbent": by_family(incumbent), - }, - }); - use std::io::Write; - if let Ok(mut file) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&self.path) - { - let _ = writeln!(file, "{line}"); - } - } -} - -/// `Σ terms ≥ run` for each run whose terms are all configuration or -/// window terms (none on the run's own indicator): at integral binaries a -/// used run has one term at 1. -fn cover_rows(milp: &SelectionMilp<'_>) -> Vec { - let mut rows = Vec::new(); - for (&(context, candidate), terms) in &milp.run_terms_cache { - if terms.is_empty() || terms.iter().any(|term| term.plain) { - continue; - } - let Some(run) = milp.run_indicator(context, candidate) else { - continue; - }; - let sum: Expression = terms.iter().map(|term| term.indicator.clone()).sum(); - rows.push(constraint!(sum >= run)); - } - rows -} diff --git a/crates/plan-selection/src/milp/mod.rs b/crates/plan-selection/src/milp/mod.rs index 7c0e9ab2f..9a2d7f7e3 100644 --- a/crates/plan-selection/src/milp/mod.rs +++ b/crates/plan-selection/src/milp/mod.rs @@ -1,26 +1,3 @@ -//! #1095 Steps 3–5: the group selection MILP (§2.0) over -//! [`MilpInputs`](crate::groups::MilpInputs): one candidate per result slot -//! per execution context, solved once with HiGHS from a warm start. -//! -//! - [`model`]: the variables and the structural rows (coverage, slots, -//! validity, used only if read, sharing, reach, reader constraints, -//! latency, accuracy), the solve and the raw solution; -//! - [`cost_terms`]: the cost model 1 objective, every coefficient a -//! number `asap-cost` computed (§3); -//! - [`warm_start`]: each query's cheapest plan, then greedy sharing, -//! HiGHS's first incumbent; -//! - [`completion`]: the exact objective of a plan given as binaries, the -//! cheapest values of the continuous variables found without a solver; -//! - [`decode`]: the solution as a `DagOfSelectedCandidates`, one node per -//! candidate run; -//! - `lp_diagnosis` (feature `lp-diagnosis`, measurement only): where the -//! LP relaxation loses its bound. -//! -//! [`crate::select`] runs them, realizes the plan once and checks it. -pub(crate) mod completion; -pub(crate) mod cost_terms; -pub(crate) mod decode; -#[cfg(feature = "lp-diagnosis")] -pub(crate) mod lp_diagnosis; +//! Stage 3's selection MILP (concepts doc, sections 3–5): its variables, +//! one function per constraint family, and the execution contexts. pub(crate) mod model; -pub(crate) mod warm_start; diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index ffd268768..5af066859 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -1,2185 +1,137 @@ -//! #1095 §2.0: the group selection MILP's variables and rows over execution -//! contexts ([`crate::groups`]), one HiGHS solve, and the raw solution. +//! Stage 3's selection MILP, named after the formulation (concepts doc, +//! sections 3–5). //! -//! ## Variables +//! | Family | Variables | Constraints | +//! |---|---|---| +//! | baseline (section 3) | $x_v$, $y_{u,c,i}$, $z_{q,v}$, $t_v$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) [`utilization_constraints`]; (G), (H): $y$ exists only on admissible edges | +//! | execution contexts (section 5) | $x^{\mathrm{mat}}_v$, $x_{\kappa,v}$, $y_{\kappa,u,c,i}$, $h_v$ ([`ExecutionContextVariables`]) | (R) [`retention_constraints`] | +//! | accuracy (section 4) | $E_{c,m}$, $F_{c,i,m}$, $D_c$, $H_{c,i}$ ([`AccuracyVariables`]) | (K) [`accuracy_constraints`] | //! -//! | Family | Meaning | -//! |---|---| -//! | `p[ctx, c]` | in context `ctx`, candidate `c` gives its result slot (binary): computed there, or read across the boundary (a shareable candidate, or a candidate of an ingestion pipeline `I_r`) | -//! | `x[m]` | shareable candidate `m` is selected (binary) | -//! | `a[q, m]` | query `q` depends on shareable `m` (continuous, exact at integral binaries) | -//! | `y[q, m, c]` | `AND(a[q, m], p[m, c])`, for reach from above and accuracy (continuous) | -//! | `L[ctx]`, `T[q, m]`, `K[q]` | latency of a query or of a query-time shareable candidate's run and context; `m`'s latency in `q`'s; compaction (continuous) | -//! | `N[q]`, `A[q, parts]` | approximate values `q` depends on; whether it depends on one sized for `parts` shares (continuous) | -//! | `g[m, c]` | highest shared materialization: `c` is read in context `m`, which at most one query reaches (binary) | -//! | `rank` | order ranks on a context's cyclic links (continuous) | -//! -//! The cost families are [`super::cost_terms`]'s. -//! -//! ## Rows (numbered as in #1095 §2.0) -//! -//! 1. Coverage: `Σ_{c ∈ Out(r)} p[q(r), c] = 1` per output reference. -//! 2. One candidate per result slot per context: `Σ_{c ∈ slot} p[ctx, c] ≤ 1`. -//! 3. Validity: `Σ_{c' ∈ compat(c, n)} p[ctx, c'] ≥ p[ctx, c]` for each `c` -//! computed in `ctx`; `≥ x[m]` for `m`'s own inputs in its context; -//! `p[ctx, m] ≤ x[m]`, `p[m, c] ≤ p[I_r(m), c]` for reads across a -//! boundary (`r(m)` the refresh window of ingestion-time `m`). -//! 4. Used only if read, per context; a candidate of `I_r` is also read -//! by `p[m, c]` for each `m` with `r(m) = r`. -//! 5. Sharing: `x[m] ≤ Σ_ctx p[ctx, m]`. -//! 6. Reach: `a[q, m] ≥ p[q, m]`, `a[q, c] ≥ a[q, m] + p[m, c] − 1`; from -//! above, where a reader constraint counts queries. -//! 7. Latency (`plan_cost`'s: each query-time run it reaches counts once). -//! 8. Accuracy (A1), per run a query depends on. -//! 9. Reader constraints. -//! -//! A link from a candidate to one of its own result slot is dropped: with -//! one candidate per slot per context it can only be a self-loop (a -//! widened scan's `Project` lists its own memo group). Links inside a -//! larger cyclic component of a context get order ranks. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::time::Instant; - -use asap_cost::{CostInputs, CostModel}; -use asap_types::stages::{ - output_references, PhysicalASAPMemoCandidates, PhysicalCandidateId, ReaderConstraint, -}; -use asap_types::workload::QueryWorkloadEntry; -use good_lp::solvers::highs::highs; -use good_lp::{ - constraint, variable, Constraint, Expression, IntoAffineExpression, ProblemVariables, - SolverModel, Variable, -}; - -use crate::groups::{AccuracyWeight, ContextKind, MilpInputs, ResultSlot, RunKind}; - -use super::cost_terms::{self, RunTerm}; -use super::warm_start; - -/// Why the selection MILP was not built or solved. -#[derive(Debug, Clone, PartialEq)] -pub(crate) enum SelectionMilpError { - /// The MILP implements cost model 1 only (§3); the name of the model given. - UnsupportedCostModel(&'static str), - /// `entries` and the query roots differ in number. - EntryCount { roots: usize, entries: usize }, - /// No plan meets every row. - Infeasible, - /// A rate or another number `asap-cost` could not compute. - Cost(String), - /// HiGHS failed. - Solver(String), -} - -/// How the MILP is built. -#[derive(Debug, Clone, Copy)] -pub(crate) struct SelectionMilpOptions { - /// Write the accuracy rows (row 8); off when the facade ignores - /// accuracy guarantees. - pub(crate) enforce_accuracy: bool, - /// Solve each independent part alone ([`independent_parts`]); off only - /// to compare with one solve in tests. - pub(crate) split_into_parts: bool, - /// HiGHS's threads (its `threads` option). One thread makes ties among - /// plans of equal cost break the same way on every run. - pub(crate) highs_threads: i32, -} +//! Every function body is `todo!()` until #1207 PRs 6 (baseline), 7 +//! (contexts) and 8 (accuracy) fill them in; until then nothing calls +//! them, hence the `dead_code` allowance below. +#![allow(dead_code)] -impl Default for SelectionMilpOptions { - fn default() -> Self { - Self { - enforce_accuracy: true, - split_into_parts: true, - highs_threads: 1, - } - } -} - -/// Which part of the model a variable or row belongs to, for the size -/// report: the query contexts, the `I_r`, the shareable candidates' contexts -/// (with their `x`, reach and schedule terms), or the rest. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)] -pub(crate) enum Scope { - Queries, - Ingestion, - SharedContexts, - Global, -} - -/// The size of one family (or scope) of variables and rows. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)] -pub(crate) struct FamilySize { - pub(crate) variables: usize, - pub(crate) binaries: usize, - pub(crate) rows: usize, - /// Nonzero coefficients of its rows. - pub(crate) nonzeros: usize, -} - -impl FamilySize { - fn add(&mut self, other: FamilySize) { - self.variables += other.variables; - self.binaries += other.binaries; - self.rows += other.rows; - self.nonzeros += other.nonzeros; - } -} +use std::collections::HashMap; -/// Variables and rows, counted per family and per scope. -pub(crate) struct ModelBuilder { - pub(crate) variables: ProblemVariables, - pub(crate) rows: Vec, - /// The family of each row of `rows`. - pub(crate) row_families: Vec<&'static str>, - pub(crate) families: BTreeMap<&'static str, FamilySize>, - pub(crate) scopes: BTreeMap, - /// Where new variables and rows are counted. - pub(crate) scope: Scope, - /// The family of each variable, in creation order. - #[cfg(feature = "lp-diagnosis")] - pub(crate) column_families: Vec<&'static str>, -} - -impl ModelBuilder { - fn new() -> Self { - Self { - variables: ProblemVariables::new(), - rows: Vec::new(), - row_families: Vec::new(), - families: BTreeMap::new(), - scopes: BTreeMap::new(), - scope: Scope::Global, - #[cfg(feature = "lp-diagnosis")] - column_families: Vec::new(), - } - } - - fn count(&mut self, family: &'static str, size: FamilySize) { - #[cfg(feature = "lp-diagnosis")] - if size.variables > 0 { - self.column_families.push(family); - } - self.families.entry(family).or_default().add(size); - self.scopes.entry(self.scope).or_default().add(size); - } - - pub(crate) fn binary(&mut self, family: &'static str) -> Variable { - self.count( - family, - FamilySize { - variables: 1, - binaries: 1, - ..FamilySize::default() - }, - ); - self.variables.add(variable().binary()) - } +use good_lp::{Constraint, Variable}; - /// A continuous variable in `[0, max]` (unbounded above for `None`). - pub(crate) fn continuous(&mut self, family: &'static str, max: Option) -> Variable { - self.count( - family, - FamilySize { - variables: 1, - ..FamilySize::default() - }, - ); - let definition = variable().min(0.0); - self.variables.add(match max { - Some(max) => definition.max(max), - None => definition, - }) - } - - pub(crate) fn row(&mut self, family: &'static str, row: Constraint) { - let nonzeros = row.expression().linear_coefficients().count(); - self.count( - family, - FamilySize { - rows: 1, - nonzeros, - ..FamilySize::default() - }, - ); - self.rows.push(row); - self.row_families.push(family); - } - - /// `and ≥ Σ terms − (n − 1)`, `and ∈ [0, 1]`: the AND of `n` - /// indicators, tight under a cost or a `≤` row that charges it. - pub(crate) fn and_at_least(&mut self, family: &'static str, terms: &[Expression]) -> Variable { - let and = self.continuous(family, Some(1.0)); - let sum: Expression = terms.iter().cloned().sum(); - let count = terms.len() as f64; - self.row(family, constraint!(and >= sum - (count - 1.0))); - and - } -} +use asap_types::accuracy::AccuracyMetric; +use asap_types::stages::{ + ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, +}; -/// One way a candidate's input is read in a context: `(reader, input)` -/// with the reader computed in that context, the query's output reference, -/// or the shareable candidate's own input. +/// A query, by index in the workload. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum Reader { - Candidate(PhysicalCandidateId, usize), - Output(usize), - OwnInput(usize), -} - -/// One input of a candidate: `(reader, input)`. -type InputOf = (PhysicalCandidateId, usize); - -/// The slot links of one multi-slot input: `(slot, e)`. -type SlotLinks = Vec<(ResultSlot, Variable)>; - -/// The built model, before the solve. -pub(crate) struct SelectionMilp<'a> { - pub(crate) builder: ModelBuilder, - pub(crate) physical: &'a PhysicalASAPMemoCandidates, - pub(crate) inputs: &'a MilpInputs, - pub(crate) entries: &'a [QueryWorkloadEntry], - pub(crate) cost_inputs: &'a CostInputs, - /// Per context, `p[ctx, c]` per option. - pub(crate) choice: Vec>, - /// Per context, per option, the reads of it in that context. - pub(crate) readers: Vec>>, - /// `x[m]` per shareable candidate with a context. - pub(crate) selected: BTreeMap, - /// `a[q, m]`, for each query that may depend on shareable `m`. - pub(crate) reach: HashMap<(usize, PhysicalCandidateId), Variable>, - /// Per shareable candidate, the shareable candidates whose contexts may - /// read it. - pub(crate) shared_readers: BTreeMap>, - /// `L[q]`. - pub(crate) latency: Vec, - /// An upper bound of each `L[q]` (its bound, or the most it can be), in - /// latency units. - pub(crate) latency_max: Vec, - /// Milliseconds per latency unit: every latency variable (`L`, `T`, `K` - /// and the held times) counts in these units, chosen so the largest - /// possible latency is about 1,000 units (with sub-microsecond work per - /// execution, milliseconds fall below HiGHS's absolute tolerances). - pub(crate) ms_per_latency_unit: f64, - /// The cost model 1 objective, in cost per second, without the request - /// fees (constant for every plan, added after selection). - pub(crate) objective_terms: Vec<(Variable, f64)>, - /// The request fees per second (the same for every plan). - pub(crate) request_cost_per_second: f64, - /// Terms encoded inexactly, by reason. - pub(crate) inexact: BTreeMap<&'static str, usize>, - /// The terms of each run `(context, candidate)`, cached; a shareable - /// candidate's own run is keyed by its own context. - pub(crate) run_terms_cache: HashMap<(usize, PhysicalCandidateId), Vec>, - /// Runs whose terms are being built (cycles of links). - pub(crate) terms_in_progress: HashSet<(usize, PhysicalCandidateId)>, - /// Per run and outputs, the variable `≥` its terms giving them. - pub(crate) producer_gives: HashMap<(usize, PhysicalCandidateId, Vec), Variable>, - /// Per query-time scan run, its window terms - /// ([`cost_terms::window_terms`]). - pub(crate) window_cache: HashMap<(usize, PhysicalCandidateId), Option>>, - /// For each input whose producers lie in several result slots (a - /// factored root over the private nodes of several implementations), - /// per context: `e[ctx, r, n, slot]`, the slot input `n` of computed - /// reader `r` reads (binary). Row 4 counts these reads, so a selected - /// producer in the other slot is not justified by `r`. - pub(crate) slot_links: - BTreeMap<(usize, PhysicalCandidateId, usize), Vec<(ResultSlot, Variable)>>, - /// [`SelectionMilpOptions::split_into_parts`]. - pub(crate) split_into_parts: bool, - /// [`SelectionMilpOptions::highs_threads`]. - pub(crate) highs_threads: i32, -} - -/// Variables and rows of one model. -#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] -pub(crate) struct SelectionMilpSize { - pub(crate) variables: usize, - pub(crate) binaries: usize, - pub(crate) rows: usize, - pub(crate) nonzeros: usize, - pub(crate) families: BTreeMap<&'static str, FamilySize>, - pub(crate) scopes: BTreeMap, - pub(crate) inexact: BTreeMap<&'static str, usize>, -} - -/// The raw solution of a solve run to proven optimality, ready to decode. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct SelectionMilpSolution { - /// The objective: cost per second without the request fees. - pub(crate) cost_per_second: f64, - /// The request fees per second, the same for every plan. - pub(crate) request_cost_per_second: f64, - /// HiGHS's dual bound on `cost_per_second`. - pub(crate) cost_lower_bound_per_second: f64, - /// Per context, the options with `p[ctx, c] = 1`. - pub(crate) chosen: Vec>, - /// The shareable candidates with `x[m] = 1`. - pub(crate) selected: BTreeSet, - /// Per multi-slot input `(context, reader, input)` of a used reader, the - /// slot it reads. - pub(crate) slot_reads: BTreeMap<(usize, PhysicalCandidateId, usize), ResultSlot>, - pub(crate) size: SelectionMilpSize, - /// Per independent part, how its solve went, in part order. - pub(crate) parts: Vec, - pub(crate) build_ms: u64, - /// Building the warm start, HiGHS's first incumbent. - pub(crate) warm_start_ms: u64, - /// The warm start's objective (like `cost_per_second`), when its - /// completion costed it. - pub(crate) warm_start_cost_per_second: Option, - pub(crate) solve_ms: u64, -} - -impl<'a> SelectionMilp<'a> { - pub(crate) fn query_count(&self) -> usize { - self.physical.query_roots.len() - } - - /// The context where `context` reads a candidate of an `I_r` - /// ([`MilpInputs::ingestion_context_read_by`]). - pub(crate) fn ingestion_context_read_by(&self, context: usize) -> Option { - self.inputs.ingestion_context_read_by(context) - } - - /// The context of shareable candidate `m`. - pub(crate) fn context_of(&self, shareable: PhysicalCandidateId) -> Option { - self.inputs.context_of_shareable.get(&shareable).copied() - } - - pub(crate) fn kind(&self, candidate: PhysicalCandidateId) -> RunKind { - self.inputs.kind[candidate.0] - } - - pub(crate) fn slot(&self, candidate: PhysicalCandidateId) -> ResultSlot { - self.inputs.slot[candidate.0] - } - - /// Whether `candidate` is computed in context `context`. - pub(crate) fn computes(&self, context: usize, candidate: PhysicalCandidateId) -> bool { - self.inputs.contexts[context].computes(self.kind(candidate)) - } - - /// The indicator of run `(context, candidate)`: `p[ctx, c]` for a - /// candidate computed there, `x[m]` for a shareable candidate's own run - /// in its own context. - pub(crate) fn run_indicator( - &self, - context: usize, - candidate: PhysicalCandidateId, - ) -> Option { - if self.inputs.contexts[context].kind == ContextKind::Shared(candidate) { - return self.selected.get(&candidate).copied(); - } - if !self.computes(context, candidate) { - return None; - } - self.choice[context].get(&candidate).copied() - } - - /// The context where run `(context, candidate)` reads its inputs: its - /// own (a shareable candidate's own run reads in its context too). - pub(crate) fn runs_at_query_time(&self, context: usize) -> bool { - self.inputs.contexts[context].at_query_time - } - - pub(crate) fn size(&self) -> SelectionMilpSize { - let families = self.builder.families.clone(); - SelectionMilpSize { - variables: families.values().map(|f| f.variables).sum(), - binaries: families.values().map(|f| f.binaries).sum(), - rows: families.values().map(|f| f.rows).sum(), - nonzeros: families.values().map(|f| f.nonzeros).sum(), - families, - scopes: self.builder.scopes.clone(), - inexact: self.inexact.clone(), - } - } - - pub(crate) fn note_inexact(&mut self, reason: &'static str) { - *self.inexact.entry(reason).or_default() += 1; - } - - /// The scope of context `context`. - pub(crate) fn scope_of(&self, context: usize) -> Scope { - match self.inputs.contexts[context].kind { - ContextKind::Query(_) => Scope::Queries, - ContextKind::Ingestion(_) => Scope::Ingestion, - ContextKind::Shared(_) => Scope::SharedContexts, - } - } - - /// Run `(context, candidate)`'s terms ([`cost_terms::run_terms`]), - /// cached. - pub(crate) fn run_terms( - &mut self, - context: usize, - candidate: PhysicalCandidateId, - ) -> Vec { - if let Some(terms) = self.run_terms_cache.get(&(context, candidate)) { - return terms.clone(); - } - let scope_here = self.scope_of(context); - let scope = std::mem::replace(&mut self.builder.scope, scope_here); - let terms = cost_terms::run_terms(self, context, candidate); - self.builder.scope = scope; - self.run_terms_cache - .insert((context, candidate), terms.clone()); - terms - } - - /// Every run of the model: `(context, candidate)` for each candidate - /// computed in a context, and each shareable candidate's own run. - pub(crate) fn runs(&self) -> Vec<(usize, PhysicalCandidateId)> { - let mut runs = Vec::new(); - for (context, options) in self.choice.iter().enumerate() { - for &candidate in options.keys() { - if self.computes(context, candidate) { - runs.push((context, candidate)); - } - } - if let ContextKind::Shared(own) = self.inputs.contexts[context].kind { - if self.selected.contains_key(&own) { - runs.push((context, own)); - } - } - } - runs - } - - /// The query contexts that may depend on shareable `m`, ascending. - pub(crate) fn queries_reaching(&self, shareable: PhysicalCandidateId) -> Vec { - let mut queries: Vec = (0..self.query_count()) - .filter(|&query| self.reach.contains_key(&(query, shareable))) - .collect(); - queries.sort_unstable(); - queries - } -} - -/// Builds the MILP over `inputs` (§2.0): `entries[q]` is query `q`'s -/// workload entry. -pub(crate) fn build_selection_milp<'a>( - physical: &'a PhysicalASAPMemoCandidates, - inputs: &'a MilpInputs, - entries: &'a [QueryWorkloadEntry], - cost_inputs: &'a CostInputs, - options: SelectionMilpOptions, -) -> Result, SelectionMilpError> { - match cost_inputs.cost_model { - CostModel::UsageOverTime(_) => {} - CostModel::ProvisionedCapacity(_) => { - return Err(SelectionMilpError::UnsupportedCostModel( - "ProvisionedCapacity (cost model 2)", - )) - } - CostModel::ClusterCapacity(_) => { - return Err(SelectionMilpError::UnsupportedCostModel( - "ClusterCapacity (cost model 3)", - )) - } - } - let query_count = physical.query_roots.len(); - if entries.len() != query_count { - return Err(SelectionMilpError::EntryCount { - roots: query_count, - entries: entries.len(), - }); - } - let mut milp = SelectionMilp { - builder: ModelBuilder::new(), - physical, - inputs, - entries, - cost_inputs, - choice: Vec::new(), - readers: Vec::new(), - selected: BTreeMap::new(), - reach: HashMap::new(), - shared_readers: BTreeMap::new(), - latency: Vec::new(), - latency_max: Vec::new(), - ms_per_latency_unit: 1.0, - objective_terms: Vec::new(), - request_cost_per_second: 0.0, - inexact: BTreeMap::new(), - run_terms_cache: HashMap::new(), - terms_in_progress: HashSet::new(), - producer_gives: HashMap::new(), - window_cache: HashMap::new(), - slot_links: BTreeMap::new(), - split_into_parts: options.split_into_parts, - highs_threads: options.highs_threads, - }; - choice_variables(&mut milp); - structural_rows(&mut milp); - acyclic_rows(&mut milp); - acyclic_context_rows(&mut milp); - reach_rows(&mut milp); - reader_constraint_rows(&mut milp); - latency_rows(&mut milp)?; - if options.enforce_accuracy { - accuracy_rows(&mut milp); - } - cost_terms::encode_objective(&mut milp)?; - Ok(milp) -} - -/// `p`, `x` and, per context, the reads of each option. -fn choice_variables(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - let output_refs = output_references(&milp.physical.query_roots); - for (context, data) in inputs.contexts.iter().enumerate() { - milp.builder.scope = milp.scope_of(context); - let options: BTreeSet = data.options.iter().copied().collect(); - let mut readers: BTreeMap> = BTreeMap::new(); - let mut choices = BTreeMap::new(); - for &option in &options { - choices.insert(option, milp.builder.binary("p (choices)")); - } - for &reader in &options { - if !data.computes(inputs.kind[reader.0]) { - continue; - } - for (input, producers) in inputs.producer_candidates_per_input[reader.0] - .iter() - .enumerate() - { - for &producer in producers { - // A read of its own result slot is a self-loop. - if options.contains(&producer) - && inputs.slot[producer.0] != inputs.slot[reader.0] - { - readers - .entry(producer) - .or_default() - .push(Reader::Candidate(reader, input)); - } - } - } - } - match data.kind { - ContextKind::Query(query) => { - for (reference, (q, _)) in output_refs.iter().enumerate() { - if *q != query { - continue; - } - for &out in &inputs.output_candidates[reference] { - if options.contains(&out) { - readers - .entry(out) - .or_default() - .push(Reader::Output(reference)); - } - } - } - } - ContextKind::Shared(own) => { - for (input, producers) in inputs.producer_candidates_per_input[own.0] - .iter() - .enumerate() - { - for &producer in producers { - if options.contains(&producer) { - readers - .entry(producer) - .or_default() - .push(Reader::OwnInput(input)); - } - } - } - } - ContextKind::Ingestion(_) => {} - } - for &reader in &options { - if !data.computes(inputs.kind[reader.0]) { - continue; - } - for (input, producers) in inputs.producer_candidates_per_input[reader.0] - .iter() - .enumerate() - { - let slots: BTreeSet = producers - .iter() - .filter(|producer| options.contains(producer)) - .map(|producer| inputs.slot[producer.0]) - .filter(|&slot| slot != inputs.slot[reader.0]) - .collect(); - if slots.len() > 1 { - let links = slots - .into_iter() - .map(|slot| (slot, milp.builder.binary("e (slot an input reads)"))) - .collect(); - milp.slot_links.insert((context, reader, input), links); - } - } - } - milp.choice.push(choices); - milp.readers.push(readers); - } - milp.builder.scope = Scope::SharedContexts; - for &shareable in inputs.context_of_shareable.keys() { - let selected = milp.builder.binary("x (shareable candidates)"); - milp.selected.insert(shareable, selected); - } - for (context, data) in inputs.contexts.iter().enumerate() { - if let ContextKind::Shared(reader) = data.kind { - for &option in &data.options { - if inputs.kind[option.0] == RunKind::Shareable { - milp.shared_readers.entry(option).or_default().push(reader); - } - } - let _ = context; - } - } - milp.builder.scope = Scope::Global; -} - -/// Rows 1–5. -fn structural_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - let output_refs = output_references(&milp.physical.query_roots); - // Per `I_r`, the contexts of the ingestion-time shareable candidates of - // refresh window `r`. - let mut readers_of_ingestion: BTreeMap> = BTreeMap::new(); - for (m, &at) in &inputs.ingestion_context_of_shareable { - readers_of_ingestion - .entry(at) - .or_default() - .push(inputs.context_of_shareable[m]); - } - // 1. Coverage. - milp.builder.scope = Scope::Queries; - for (reference, (query, _)) in output_refs.iter().enumerate() { - let sum: Expression = inputs.output_candidates[reference] - .iter() - .filter_map(|out| milp.choice[*query].get(out).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("1 coverage", constraint!(sum == 1.0)); - } - for context in 0..inputs.contexts.len() { - milp.builder.scope = milp.scope_of(context); - let data = &inputs.contexts[context]; - // 2. One candidate per slot. - let mut per_slot: BTreeMap> = BTreeMap::new(); - for (&option, &variable) in &milp.choice[context] { - per_slot - .entry(inputs.slot[option.0]) - .or_default() - .push(variable); - } - for variables in per_slot.values().filter(|v| v.len() > 1) { - let sum: Expression = variables.iter().copied().map(Expression::from).sum(); - milp.builder - .row("2 one candidate per slot", constraint!(sum <= 1.0)); - } - // 3. Validity. - let choices = milp.choice[context].clone(); - let producers_sum = |reader: PhysicalCandidateId, input: usize| -> Expression { - inputs.producer_candidates_per_input[reader.0][input] - .iter() - .filter(|producer| inputs.slot[producer.0] != inputs.slot[reader.0]) - .filter_map(|producer| choices.get(producer).copied()) - .map(Expression::from) - .sum() - }; - for (&option, &variable) in &choices { - let kind = inputs.kind[option.0]; - if data.computes(kind) { - for input in 0..inputs.producer_candidates_per_input[option.0].len() { - let sum = producers_sum(option, input); - milp.builder.row("3 validity", constraint!(sum >= variable)); - } - } else { - match kind { - RunKind::Shareable => { - let selected = milp.selected[&option]; - milp.builder - .row("3 validity (boundary)", constraint!(variable <= selected)); - } - RunKind::Ingestion => { - let source = milp - .ingestion_context_read_by(context) - .and_then(|ingestion| milp.choice[ingestion].get(&option).copied()); - match source { - Some(source) => milp - .builder - .row("3 validity (boundary)", constraint!(variable <= source)), - None => milp - .builder - .row("3 validity (boundary)", constraint!(variable <= 0.0)), - } - } - RunKind::PerContext => milp - .builder - .row("3 validity (boundary)", constraint!(variable <= 0.0)), - } - } - } - if let ContextKind::Shared(own) = data.kind { - let selected = milp.selected[&own]; - for input in 0..inputs.producer_candidates_per_input[own.0].len() { - let sum: Expression = inputs.producer_candidates_per_input[own.0][input] - .iter() - .filter_map(|producer| choices.get(producer).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("3 validity", constraint!(sum >= selected)); - } - } - // The slot each multi-slot input reads: one, and only a slot the - // context gives. - let links: Vec<(InputOf, SlotLinks)> = milp - .slot_links - .range((context, PhysicalCandidateId(0), 0)..(context + 1, PhysicalCandidateId(0), 0)) - .map(|(&(_, reader, input), links)| ((reader, input), links.clone())) - .collect(); - for ((reader, input), links) in links { - let used = choices[&reader]; - let sum: Expression = links.iter().map(|&(_, link)| Expression::from(link)).sum(); - milp.builder - .row("3 validity (slot links)", constraint!(sum == used)); - for (slot, link) in links { - let given: Expression = inputs.producer_candidates_per_input[reader.0][input] - .iter() - .filter(|producer| inputs.slot[producer.0] == slot) - .filter_map(|producer| choices.get(producer).copied()) - .map(Expression::from) - .sum(); - milp.builder - .row("3 validity (slot links)", constraint!(link <= given)); - } - } - // 4. Used only if read. - for (&option, &variable) in &choices { - let mut reads = Expression::from(0.0); - for reader in milp.readers[context].get(&option).into_iter().flatten() { - match *reader { - Reader::Candidate(reader, input) => { - match milp.slot_links.get(&(context, reader, input)) { - Some(links) => { - let slot = inputs.slot[option.0]; - for &(at, link) in links { - if at == slot { - reads += link; - } - } - } - None => reads += choices[&reader], - } - } - Reader::Output(_) => reads += 1.0, - Reader::OwnInput(_) => { - if let ContextKind::Shared(own) = data.kind { - reads += milp.selected[&own]; - } - } - } - } - // A candidate of `I_r` is also read by the contexts of the - // ingestion-time shareable candidates of refresh window `r`. - for &other in readers_of_ingestion.get(&context).into_iter().flatten() { - if let Some(&read) = milp.choice[other].get(&option) { - reads += read; - } - } - milp.builder - .row("4 used only if read", constraint!(variable <= reads)); - } - } - // 5. Sharing. - milp.builder.scope = Scope::SharedContexts; - let mut reads_of: BTreeMap = BTreeMap::new(); - for choices in &milp.choice { - for (&option, &variable) in choices { - if inputs.kind[option.0] == RunKind::Shareable { - *reads_of.entry(option).or_default() += variable; - } - } - } - let selected = milp.selected.clone(); - for (shareable, selected) in selected { - let reads = reads_of.remove(&shareable).unwrap_or_default(); - milp.builder - .row("5 sharing", constraint!(selected <= reads)); - } - milp.builder.scope = Scope::Global; -} - -/// Order ranks on the links inside each cyclic component of a context's -/// links between candidates it computes (none when the slot graph is -/// acyclic). -fn acyclic_rows(milp: &mut SelectionMilp<'_>) { - for context in 0..milp.choice.len() { - milp.builder.scope = milp.scope_of(context); - let computed: Vec = milp.choice[context] - .keys() - .copied() - .filter(|&c| milp.computes(context, c)) - .collect(); - let index: HashMap = - computed.iter().enumerate().map(|(i, &c)| (c, i)).collect(); - let mut edges: Vec<(usize, usize)> = Vec::new(); - for (&producer, readers) in &milp.readers[context] { - let Some(&from) = index.get(&producer) else { - continue; - }; - for reader in readers { - if let Reader::Candidate(reader, _) = reader { - edges.push((from, index[reader])); - } - } - } - let components = strongly_connected_components(computed.len(), &edges); - let mut rank: HashMap = HashMap::new(); - for &(producer, reader) in &edges { - let size = components.size_of(reader); - if components.of[producer] != components.of[reader] || size < 2 { - continue; - } - let n = size as f64; - let [producer_rank, reader_rank] = [producer, reader].map(|node| { - *rank - .entry(node) - .or_insert_with(|| milp.builder.continuous("acyclic ranks", Some(n - 1.0))) - }); - // Both used: the reader ranks above the producer. (Within one - // context a used reader reads the used candidate of the slot.) - let (p, r) = ( - milp.choice[context][&computed[producer]], - milp.choice[context][&computed[reader]], - ); - milp.builder.row( - "acyclic ranks", - constraint!(producer_rank + 1.0 <= reader_rank + n * (2.0 - p - r)), - ); - } - } - milp.builder.scope = Scope::Global; -} - -/// Order ranks across contexts: on the graph whose nodes are the shareable -/// candidates' own runs and the runs of each `I_r`, with an edge from a run -/// to each shareable candidate or candidate of an `I_r` its context may -/// read (for a run of `I_r`, its producers there). Rows 3–5 alone admit a -/// cycle of reads across contexts that justifies itself with no output -/// above it (two shareable candidates of one memo group, each listed among -/// the other's producers); only edges inside a cyclic component get a row. -fn acyclic_context_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - // Nodes: (context of the run, candidate). - let mut nodes: Vec<(usize, PhysicalCandidateId)> = Vec::new(); - for (&m, &context) in &inputs.context_of_shareable { - nodes.push((context, m)); - } - for (context, data) in inputs.contexts.iter().enumerate() { - if let ContextKind::Ingestion(_) = data.kind { - for &c in milp.choice[context].keys() { - if milp.computes(context, c) { - nodes.push((context, c)); - } - } - } - } - let index: HashMap<(usize, PhysicalCandidateId), usize> = - nodes.iter().enumerate().map(|(i, &run)| (run, i)).collect(); - // The node a read of `option` in `context` resolves to. - let node_read = |context: usize, option: PhysicalCandidateId| -> Option { - let run_context = match inputs.kind[option.0] { - RunKind::Shareable => milp.context_of(option)?, - RunKind::Ingestion => milp.ingestion_context_read_by(context)?, - RunKind::PerContext => return None, - }; - index.get(&(run_context, option)).copied() - }; - // Edges with their activity: `(from, to, indicators that must all be 1)`. - let mut edges: Vec<(usize, usize, Vec)> = Vec::new(); - for (from, &(context, candidate)) in nodes.iter().enumerate() { - if let ContextKind::Ingestion(_) = inputs.contexts[context].kind { - let reader = milp.choice[context][&candidate]; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - for &producer in producers { - if let (Some(to), Some(&read)) = ( - node_read(context, producer), - milp.choice[context].get(&producer), - ) { - if to != from { - edges.push((from, to, vec![reader, read])); - } - } - } - } - } else { - for (&option, &read) in &milp.choice[context] { - if let Some(to) = node_read(context, option) { - edges.push((from, to, vec![read])); - } - } - } - } - let plain: Vec<(usize, usize)> = edges.iter().map(|&(from, to, _)| (from, to)).collect(); - let components = strongly_connected_components(nodes.len(), &plain); - milp.builder.scope = Scope::SharedContexts; - let mut rank: HashMap = HashMap::new(); - for (from, to, active) in edges { - let size = components.size_of(from); - if components.of[from] != components.of[to] || size < 2 { - continue; - } - let n = size as f64; - let [reader_rank, read_rank] = [from, to].map(|node| { - *rank.entry(node).or_insert_with(|| { - milp.builder - .continuous("acyclic ranks (contexts)", Some(n - 1.0)) - }) - }); - let slack = active.len() as f64; - let on: Expression = active.into_iter().map(Expression::from).sum(); - milp.builder.row( - "acyclic ranks (contexts)", - constraint!(read_rank + 1.0 <= reader_rank + n * (slack - on)), - ); - } - milp.builder.scope = Scope::Global; -} - -/// The strongly connected components of a directed graph. -pub(crate) struct Components { - /// Per node, its component. - pub(crate) of: Vec, - /// Per component, its number of nodes. - pub(crate) sizes: Vec, -} - -impl Components { - pub(crate) fn size_of(&self, node: usize) -> usize { - self.sizes[self.of[node]] - } -} - -/// The strongly connected components of the graph on nodes `0..count` with -/// `edges` `(from, to)` (Tarjan's algorithm, without recursion). -pub(crate) fn strongly_connected_components(count: usize, edges: &[(usize, usize)]) -> Components { - let mut successors = vec![Vec::new(); count]; - for &(from, to) in edges { - successors[from].push(to); - } - const UNVISITED: usize = usize::MAX; - let mut order = vec![UNVISITED; count]; - let mut lowest = vec![0; count]; - let mut on_stack = vec![false; count]; - let mut stack = Vec::new(); - let mut of = vec![UNVISITED; count]; - let mut sizes = Vec::new(); - let mut next = 0; - for root in 0..count { - if order[root] != UNVISITED { - continue; - } - let mut path = vec![(root, 0)]; - order[root] = next; - lowest[root] = next; - next += 1; - stack.push(root); - on_stack[root] = true; - while let Some(&mut (node, ref mut successor)) = path.last_mut() { - if let Some(&to) = successors[node].get(*successor) { - *successor += 1; - if order[to] == UNVISITED { - order[to] = next; - lowest[to] = next; - next += 1; - stack.push(to); - on_stack[to] = true; - path.push((to, 0)); - } else if on_stack[to] { - lowest[node] = lowest[node].min(order[to]); - } - continue; - } - path.pop(); - if let Some(&(parent, _)) = path.last() { - lowest[parent] = lowest[parent].min(lowest[node]); - } - if lowest[node] == order[node] { - let component = sizes.len(); - let mut size = 0; - loop { - let member = stack.pop().expect("the node is on the stack"); - on_stack[member] = false; - of[member] = component; - size += 1; - if member == node { - break; - } - } - sizes.push(size); - } - } - } - Components { of, sizes } -} - -/// The shareable candidates in order: each before every shareable -/// candidate its context may read (readers first), so `a[q, m]` exists -/// before the reach rows of what `m` reads. A cycle among contexts is -/// broken at the first candidate met. -pub(crate) fn shareables_readers_first(milp: &SelectionMilp<'_>) -> Vec { - let inputs = milp.inputs; - let mut order = Vec::new(); - let mut state: BTreeMap = BTreeMap::new(); - // Post-order over "m reads c" (children = what m reads), reversed. - for &start in inputs.context_of_shareable.keys() { - if state.contains_key(&start) { - continue; - } - let mut stack: Vec<(PhysicalCandidateId, bool)> = vec![(start, false)]; - while let Some((node, done)) = stack.pop() { - if done { - state.insert(node, 2); - order.push(node); - continue; - } - if state.contains_key(&node) { - continue; - } - state.insert(node, 1); - stack.push((node, true)); - let context = inputs.context_of_shareable[&node]; - for &option in &inputs.contexts[context].options { - if inputs.kind[option.0] == RunKind::Shareable && !state.contains_key(&option) { - stack.push((option, false)); - } - } - } - } - order.reverse(); - order -} - -/// Row 6: reach, from below for every `(q, m)`, from above where a reader -/// constraint counts queries. -fn reach_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - milp.builder.scope = Scope::SharedContexts; - let order = shareables_readers_first(milp); - // Which queries may depend on each shareable candidate: read in their - // context, or in the context of one they may depend on. - let mut may_reach: BTreeMap> = BTreeMap::new(); - for query in 0..milp.query_count() { - for &option in &inputs.contexts[query].options { - if inputs.kind[option.0] == RunKind::Shareable { - may_reach.entry(option).or_default().insert(query); - } - } - } - for &shareable in &order { - let queries = may_reach.get(&shareable).cloned().unwrap_or_default(); - let context = inputs.context_of_shareable[&shareable]; - for &option in &inputs.contexts[context].options { - if inputs.kind[option.0] == RunKind::Shareable { - may_reach - .entry(option) - .or_default() - .extend(queries.iter().copied()); - } - } - } - for &shareable in &order { - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.builder.continuous("a (reach)", Some(1.0)); - milp.reach.insert((query, shareable), reach); - } - } - for &shareable in &order { - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.reach[&(query, shareable)]; - if let Some(&direct) = milp.choice[query].get(&shareable) { - milp.builder - .row("6 reach from below", constraint!(reach >= direct)); - } - for &reader in milp.shared_readers.get(&shareable).into_iter().flatten() { - let Some(&above) = milp.reach.get(&(query, reader)) else { - continue; - }; - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&shareable]; - milp.builder.row( - "6 reach from below", - constraint!(reach >= above + read - 1.0), - ); - } - } - } - // From above, where needed. - let exact = shareables_needing_exact_reach(milp); - for &shareable in &order { - if !exact.contains(&shareable) { - continue; - } - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.reach[&(query, shareable)]; - let mut sources = Expression::from(0.0); - if let Some(&direct) = milp.choice[query].get(&shareable) { - sources += direct; - } - for &reader in milp - .shared_readers - .get(&shareable) - .cloned() - .unwrap_or_default() - .iter() - { - let Some(&above) = milp.reach.get(&(query, reader)) else { - continue; - }; - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&shareable]; - let through = milp.builder.continuous("y (reach through)", Some(1.0)); - milp.builder - .row("6 reach from above", constraint!(through <= above)); - milp.builder - .row("6 reach from above", constraint!(through <= read)); - sources += through; - } - milp.builder - .row("6 reach from above", constraint!(reach <= sources)); - } - } - milp.builder.scope = Scope::Global; -} - -/// The reader constraints of each candidate. -pub(crate) fn reader_constraints_of_candidates<'a>( - milp: &SelectionMilp<'a>, -) -> BTreeMap> { - let physical: &'a PhysicalASAPMemoCandidates = milp.physical; - let mut of_candidate: BTreeMap> = - BTreeMap::new(); - for constraint in &physical.reader_constraints { - of_candidate - .entry(constraint.candidate()) - .or_default() - .push(constraint); - } - of_candidate -} - -/// The shareable candidates whose `a` must be exact from above: those a -/// counting reader constraint (minimum readers, highest shared -/// materialization) names or whose context holds a constrained candidate, -/// and transitively every shareable candidate whose context reads one. -fn shareables_needing_exact_reach(milp: &SelectionMilp<'_>) -> BTreeSet { - let inputs = milp.inputs; - let constraints = reader_constraints_of_candidates(milp); - let mut stack: Vec = Vec::new(); - for (candidate, constraints) in &constraints { - let counted = constraints.iter().any(|constraint| { - matches!( - constraint, - ReaderConstraint::MinimumQueryReaders { .. } - | ReaderConstraint::HighestSharedQueryMaterialization { .. } - ) - }); - if !counted { - continue; - } - if inputs.kind[candidate.0] == RunKind::Shareable { - stack.push(*candidate); - // Highest shared materialization reads the reach of its - // reading contexts. - stack.extend(milp.shared_readers.get(candidate).into_iter().flatten()); - } - for (context, data) in inputs.contexts.iter().enumerate() { - if let ContextKind::Shared(own) = data.kind { - if milp.choice[context].contains_key(candidate) { - stack.push(own); - } - } - } - } - let mut exact = BTreeSet::new(); - while let Some(shareable) = stack.pop() { - if exact.insert(shareable) { - stack.extend(milp.shared_readers.get(&shareable).into_iter().flatten()); - } - } - exact -} - -/// The queries of `query_outputs` buckets (one per query, each its output -/// references). -fn bucket_queries(query_of_reference: &[usize], query_outputs: &[Vec]) -> Vec { - query_outputs - .iter() - .filter_map(|bucket| { - bucket - .first() - .map(|&reference| query_of_reference[reference]) - }) - .collect() -} - -/// Row 9: reader constraints, on every node of the constrained candidate. -/// A node in a query's context is reached by that query alone; a node in -/// shareable `m`'s context, and `m` itself, by the queries that depend on -/// `m`. Reads into an `I_r` and per-output-reference reach inside one query -/// are not modeled: the constraint is then applied to every reference of -/// the query (conservative, counted as inexact). -fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - let query_of_reference: Vec = output_references(&milp.physical.query_roots) - .into_iter() - .map(|(query, _)| query) - .collect(); - let constraints = reader_constraints_of_candidates(milp); - let query_count = milp.query_count(); - milp.builder.scope = Scope::Global; - let reach_count = |milp: &SelectionMilp<'_>, queries: &[usize], m: PhysicalCandidateId| { - let mut count = Expression::from(0.0); - let mut most = 0.0f64; - for &query in queries { - if let Some(&reach) = milp.reach.get(&(query, m)) { - count += reach; - most += 1.0; - } - } - (count, most) - }; - for (candidate, constraints) in constraints { - let shareable = inputs.kind[candidate.0] == RunKind::Shareable; - // Its uses: (context, p) per context holding it as an option it - // computes. - let uses: Vec<(usize, Variable)> = (0..milp.choice.len()) - .filter(|&context| milp.computes(context, candidate)) - .filter_map(|context| { - milp.choice[context] - .get(&candidate) - .map(|&variable| (context, variable)) - }) - .collect(); - for constraint in constraints { - match constraint { - ReaderConstraint::MinimumQueryReaders { - query_outputs, - minimum, - .. - } => { - let queries = bucket_queries(&query_of_reference, query_outputs); - let minimum = *minimum as f64; - if shareable { - if let Some(&selected) = milp.selected.get(&candidate) { - let (count, _) = reach_count(milp, &queries, candidate); - milp.builder.row( - "9 reader constraints", - constraint!(count >= minimum * selected), - ); - } - } - for &(context, used) in &uses { - let count = match inputs.contexts[context].kind { - ContextKind::Query(_) => Expression::from(1.0), - ContextKind::Shared(m) => reach_count(milp, &queries, m).0, - ContextKind::Ingestion(_) => { - milp.note_inexact( - "minimum readers of a candidate of an I_r (forbidden)", - ); - Expression::from(0.0) - } - }; - milp.builder - .row("9 reader constraints", constraint!(count >= minimum * used)); - } - } - ReaderConstraint::HighestSharedQueryMaterialization { query_outputs, .. } => { - let queries = bucket_queries(&query_of_reference, query_outputs); - if shareable { - let Some(&selected) = milp.selected.get(&candidate) else { - continue; - }; - // Allowed: a read in a query's context (its readers - // there are reached by that query alone), or a read - // in the context of `m` that at most one query - // reaches. - let mut allowed = Expression::from(0.0); - for query in 0..query_count { - if let Some(&read) = milp.choice[query].get(&candidate) { - allowed += read; - } - } - for &reader in milp - .shared_readers - .get(&candidate) - .cloned() - .unwrap_or_default() - .iter() - { - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&candidate]; - let (count, most) = reach_count(milp, &queries, reader); - let single = milp.builder.binary("g (single-query readers)"); - milp.builder - .row("9 reader constraints", constraint!(single <= read)); - let slack = (most - 1.0).max(0.0); - milp.builder.row( - "9 reader constraints", - constraint!(count <= 1.0 + slack * (1.0 - single)), - ); - allowed += single; - } - milp.builder - .row("9 reader constraints", constraint!(selected <= allowed)); - } - for &(context, used) in &uses { - match inputs.contexts[context].kind { - ContextKind::Query(_) => {} - ContextKind::Shared(m) => { - let (count, most) = reach_count(milp, &queries, m); - let slack = (most - 1.0).max(0.0); - milp.builder.row( - "9 reader constraints", - constraint!(count <= 1.0 + slack * (1.0 - used)), - ); - } - ContextKind::Ingestion(_) => { - milp.note_inexact( - "highest shared materialization in an I_r (forbidden)", - ); - milp.builder - .row("9 reader constraints", constraint!(used <= 0.0)); - } - } - } - } - ReaderConstraint::AllowedOutputs { outputs, .. } => { - for query in 0..query_count { - let references: Vec = (0..query_of_reference.len()) - .filter(|&reference| query_of_reference[reference] == query) - .collect(); - let disallowed = references - .iter() - .filter(|reference| !outputs.contains(reference)) - .count(); - if disallowed == 0 { - continue; - } - if disallowed < references.len() { - milp.note_inexact("AllowedOutputs over part of a query's outputs"); - } - forbid_reach(milp, query, candidate, &uses); - } - } - ReaderConstraint::IncompatibleOutputs { left, right, .. } => { - let (left, right) = (query_of_reference[*left], query_of_reference[*right]); - if left == right { - milp.note_inexact("IncompatibleOutputs within one query (forbidden)"); - forbid_reach(milp, left, candidate, &uses); - continue; - } - if shareable { - if let (Some(&l), Some(&r)) = ( - milp.reach.get(&(left, candidate)), - milp.reach.get(&(right, candidate)), - ) { - milp.builder - .row("9 reader constraints", constraint!(l + r <= 1.0)); - } - } - for &(context, used) in &uses { - if let ContextKind::Shared(m) = inputs.contexts[context].kind { - if let (Some(&l), Some(&r)) = - (milp.reach.get(&(left, m)), milp.reach.get(&(right, m))) - { - milp.builder - .row("9 reader constraints", constraint!(l + r + used <= 2.0)); - } - } - } - } - } - } - } -} - -/// Query `query` reaches no node of `candidate`. -fn forbid_reach( - milp: &mut SelectionMilp<'_>, - query: usize, - candidate: PhysicalCandidateId, - uses: &[(usize, Variable)], -) { - if let Some(&reach) = milp.reach.get(&(query, candidate)) { - milp.builder - .row("9 reader constraints", constraint!(reach <= 0.0)); - } else if let Some(&read) = milp.choice[query].get(&candidate) { - milp.builder - .row("9 reader constraints", constraint!(read <= 0.0)); - } - for &(context, used) in uses { - match milp.inputs.contexts[context].kind { - ContextKind::Query(q) if q == query => { - milp.builder - .row("9 reader constraints", constraint!(used <= 0.0)); - } - ContextKind::Query(_) => {} - ContextKind::Shared(m) => { - if let Some(&reach) = milp.reach.get(&(query, m)) { - milp.builder - .row("9 reader constraints", constraint!(reach + used <= 1.0)); - } - } - ContextKind::Ingestion(_) => { - milp.note_inexact("a reader constraint on a candidate of an I_r (forbidden)"); - milp.builder - .row("9 reader constraints", constraint!(used <= 0.0)); - } - } - } -} - -/// Query `query`'s latency bound: the smaller of its `max_latency_ms` and -/// its fixed repeat interval (`checks::latency_violation`). -pub(crate) fn latency_bound_ms(entry: &QueryWorkloadEntry) -> Option { - let interval = entry.recurrence.fixed_interval_ms().map(|ms| ms as f64); - match (entry.requirements.max_latency_ms, interval) { - (Some(bound), Some(interval)) => Some(bound.min(interval)), - (bound, interval) => bound.or(interval), - } -} - -/// The latency terms of context `context`'s computed runs (and of `m`'s -/// own run in its own context): `Σ d · indicator`, and the most it can be. -fn context_work(milp: &mut SelectionMilp<'_>, context: usize) -> (Expression, f64) { - let mut work = Expression::from(0.0); - let mut most = 0.0; - let mut runs: Vec = milp.choice[context] - .keys() - .copied() - .filter(|&c| milp.computes(context, c)) - .collect(); - if let ContextKind::Shared(own) = milp.inputs.contexts[context].kind { - runs.push(own); - } - // Per slot, only the slowest candidate can count (one per slot). - let mut most_per_slot: BTreeMap = BTreeMap::new(); - for candidate in runs { - let mut run_most = 0.0; - for term in milp.run_terms(context, candidate) { - if term.numbers.latency_ms > 0.0 { - let units = term.numbers.latency_ms / milp.ms_per_latency_unit; - work += units * term.indicator.clone(); - run_most = f64::max(run_most, units); - } - } - let slot = milp.slot(candidate); - let entry = most_per_slot.entry(slot).or_default(); - *entry = entry.max(run_most); - } - most += most_per_slot.values().sum::(); - (work, most) -} - -/// Row 7: each query's latency, `asap_cost::plan_cost`'s definition: its -/// request time, plus the work of every query-time run it reaches, each -/// once (its own context's, and each query-time shareable candidate's run -/// and context through `T[q, m]`), plus the longest compaction among the -/// ingestion-time states it reaches. -fn latency_rows(milp: &mut SelectionMilp<'_>) -> Result<(), SelectionMilpError> { - let inputs = milp.inputs; - let query_count = milp.query_count(); - let request_ms = milp.cost_inputs.measured_operation_costs.ns_per_evaluation * 1e-6; - // The latency unit: the most any query's latency can be (every - // candidate's work once, plus the request) is about 1,000 units. - let most_ms = request_ms - + inputs - .candidate_data - .iter() - .filter_map(|data| data.usage.as_ref().ok()) - .map(|usage| usage.own_latency_ms) - .sum::(); - if most_ms.is_finite() && most_ms > 0.0 { - milp.ms_per_latency_unit = most_ms / 1_000.0; - } - let unit = milp.ms_per_latency_unit; - let request = request_ms / unit; - // `L[m]` per query-time shareable candidate. - milp.builder.scope = Scope::SharedContexts; - let mut shared_latency: BTreeMap = BTreeMap::new(); - let shareables: Vec<(PhysicalCandidateId, usize)> = inputs - .context_of_shareable - .iter() - .map(|(&m, &context)| (m, context)) - .collect(); - for (m, context) in shareables { - if !milp.runs_at_query_time(context) { - continue; - } - let (work, most) = context_work(milp, context); - milp.builder.scope = Scope::SharedContexts; - if most <= 0.0 { - continue; - } - let latency = milp.builder.continuous("L (latency of m)", None); - milp.builder.row("7 latency", constraint!(latency >= work)); - shared_latency.insert(m, (latency, most)); - } - let compaction = cost_terms::compaction_ms(milp); - for query in 0..query_count { - let (mut work, mut most) = context_work(milp, query); - milp.builder.scope = Scope::Queries; - work += request; - most += request; - let mut compactions = Vec::new(); - let mut longest_compaction = 0.0f64; - for (&m, &(shared, shared_most)) in &shared_latency { - let Some(&reach) = milp.reach.get(&(query, m)) else { - continue; - }; - let through = milp.builder.continuous("T (latency of m in q)", None); - milp.builder.row( - "7 latency", - constraint!(through >= shared - shared_most * (1.0 - reach)), - ); - work += through; - most += shared_most; - } - for (&m, &ms) in &compaction { - if let Some(&reach) = milp.reach.get(&(query, m)) { - if ms > 0.0 { - compactions.push((ms / unit, reach)); - longest_compaction = longest_compaction.max(ms / unit); - } - } - } - if !compactions.is_empty() { - let longest = milp.builder.continuous("K (compaction)", None); - for (ms, reach) in compactions { - milp.builder - .row("7 latency", constraint!(longest >= ms * reach)); - } - work += longest; - most += longest_compaction; - } - let latency = milp.builder.continuous("L (latency of q)", None); - let bound = latency_bound_ms(&milp.entries[query]) - .map(|bound| bound / unit) - .filter(|&bound| bound < most); - // A binding bound's rows are divided by it, so HiGHS's absolute - // feasibility tolerance is relative to the bound. - let per_ms = bound.map_or(1.0, |bound| 1.0 / bound.max(1e-9)); - milp.builder - .row("7 latency", constraint!(per_ms * latency >= per_ms * work)); - if let Some(bound) = bound { - milp.builder - .row("7 latency", constraint!(per_ms * latency <= per_ms * bound)); - } - let bound = latency_bound_ms(&milp.entries[query]).map(|bound| bound / unit); - milp.latency.push(latency); - milp.latency_max - .push(bound.map_or(most, |bound| bound.min(most))); - } - milp.builder.scope = Scope::Global; - Ok(()) -} - -/// Row 8: accuracy (A1, revised 2026-10-31), per run `q` depends on: a run -/// unusable for `q` is never depended on, two whose pair is excluded are -/// not depended on together, and the shares depended on are at most each -/// such run's `parts`. -fn accuracy_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - milp.builder.scope = Scope::Global; - let approximate_runs: Vec<(usize, PhysicalCandidateId)> = milp - .runs() - .into_iter() - .filter(|&(_, candidate)| inputs.candidate_data[candidate.0].approximate) - .collect(); - for query in 0..milp.query_count() { - let exact_target = milp.entries[query].requirements.accuracy.is_exact(); - // (candidate, dep, parts) per run `q` may depend on. - let mut counted: Vec<(PhysicalCandidateId, Expression, usize)> = Vec::new(); - // Where each counted run gives its value: (context, slot), `None` - // for a shareable candidate's own run. One value per place. - let mut places: BTreeSet<(usize, Option)> = BTreeSet::new(); - for &(context, candidate) in &approximate_runs { - let Some(weight) = inputs.accuracy_weights[query].get(&candidate).copied() else { - continue; - }; - let dep: Option = match inputs.contexts[context].kind { - ContextKind::Query(q) if q == query => { - milp.choice[context].get(&candidate).map(|&p| p.into()) - } - ContextKind::Query(_) => None, - ContextKind::Shared(own) if own == candidate => { - milp.reach.get(&(query, candidate)).map(|&a| a.into()) - } - ContextKind::Shared(own) => match milp.reach.get(&(query, own)) { - Some(&above) => { - let p = milp.choice[context][&candidate]; - Some( - milp.builder - .and_at_least( - "y (depends on a run in m)", - &[above.into(), p.into()], - ) - .into(), - ) - } - None => None, - }, - ContextKind::Ingestion(_) => { - milp.note_inexact("an approximate candidate of an I_r (forbidden)"); - let p = milp.choice[context][&candidate]; - milp.builder.row("8 accuracy", constraint!(p <= 0.0)); - None - } - }; - let Some(dep) = dep else { - continue; - }; - let usable = match weight { - _ if exact_target => false, - AccuracyWeight::One => true, - AccuracyWeight::Unusable(_) => false, - }; - if usable { - places.insert(match inputs.contexts[context].kind { - ContextKind::Shared(own) if own == candidate => (context, None), - _ => (context, Some(inputs.slot[candidate.0])), - }); - counted.push(( - candidate, - dep, - inputs.candidate_data[candidate.0].accuracy_budget_parts, - )); - } else { - milp.builder.row("8 accuracy", constraint!(dep <= 0.0)); - } - } - let mut pairs: Vec<(Expression, Expression)> = Vec::new(); - for &(first, second) in &inputs.accuracy_excluded_pairs[query] { - for (_, one, _) in counted.iter().filter(|(c, _, _)| *c == first) { - for (_, other, _) in counted.iter().filter(|(c, _, _)| *c == second) { - pairs.push((one.clone(), other.clone())); - } - } - } - for (one, other) in pairs { - milp.builder - .row("8 accuracy (pair)", constraint!(one + other <= 1.0)); - } - // The most values `q` can depend on together. - let most = if tests_loose_big_m() { - counted.len() as f64 - } else { - places.len() as f64 - }; - if counted.iter().all(|&(_, _, parts)| parts as f64 >= most) { - continue; - } - let shares = milp.builder.continuous("N (approximate values)", None); - let sum: Expression = counted.iter().map(|(_, dep, _)| dep.clone()).sum(); - milp.builder.row("8 accuracy", constraint!(shares == sum)); - let mut by_parts: BTreeMap> = BTreeMap::new(); - for (_, dep, parts) in &counted { - if (*parts as f64) < most { - by_parts.entry(*parts).or_default().push(dep.clone()); - } - } - for (parts, deps) in by_parts { - let reads = milp - .builder - .continuous("A (depends at a parts value)", Some(1.0)); - for dep in deps { - milp.builder.row("8 accuracy", constraint!(reads >= dep)); - } - let slack = most - parts as f64; - milp.builder.row( - "8 accuracy", - constraint!(shares <= parts as f64 + slack * (1.0 - reads)), - ); - } - } -} - -/// One independent part of the model (#1095 §5, 3a): a connected component -/// of the graph joining the variables of each row, or all components -/// without a query together. Parts share no variable and no row, and the -/// objective is a sum over variables, so solving each alone and joining -/// their solutions solves the whole model. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct IndependentPart { - /// The queries whose output references it covers, ascending. - pub(crate) queries: Vec, - /// Its columns (variables in creation order), ascending. - pub(crate) columns: Vec, - /// Its rows, ascending. - pub(crate) rows: Vec, -} - -/// The independent parts of `milp`, in order of their first query (the -/// part without a query last). Queries fall into one part when they may -/// share a shareable candidate, a run of an `I_r`, a source's raw retention or -/// anything else a row joins. -pub(crate) fn independent_parts(milp: &SelectionMilp<'_>) -> Vec { - let column_of: HashMap = milp - .builder - .variables - .iter_variables_with_def() - .enumerate() - .map(|(column, (variable, _))| (variable, column)) - .collect(); - let count = column_of.len(); - // Union-find over the columns, joined along each row. - let mut parent: Vec = (0..count).collect(); - fn root(parent: &mut [usize], mut node: usize) -> usize { - while parent[node] != node { - parent[node] = parent[parent[node]]; - node = parent[node]; - } - node - } - for row in &milp.builder.rows { - let mut first: Option = None; - for (variable, _) in row.expression().linear_coefficients() { - let column = root(&mut parent, column_of[&variable]); - match first { - None => first = Some(column), - Some(known) if known != column => parent[column] = known, - Some(_) => {} - } - } - } - let output_refs = output_references(&milp.physical.query_roots); - let mut query_of_component: BTreeMap = BTreeMap::new(); - for (reference, &(query, _)) in output_refs.iter().enumerate() { - for out in &milp.inputs.output_candidates[reference] { - if let Some(variable) = milp.choice[query].get(out) { - let component = root(&mut parent, column_of[variable]); - query_of_component.entry(component).or_insert(query); - } - } - } - // The part of each component: its first query's, or the last. - let part_of = |component: usize| query_of_component.get(&component).copied(); - let mut parts: BTreeMap, IndependentPart> = BTreeMap::new(); - let mut part_of_column = Vec::with_capacity(count); - for column in 0..count { - let key = part_of(root(&mut parent, column)); - part_of_column.push(key); - parts - .entry(key) - .or_insert_with(|| IndependentPart { - queries: Vec::new(), - columns: Vec::new(), - rows: Vec::new(), - }) - .columns - .push(column); - } - for (reference, &(query, _)) in output_refs.iter().enumerate() { - let Some(variable) = milp.inputs.output_candidates[reference] - .iter() - .find_map(|out| milp.choice[query].get(out)) - else { - continue; - }; - let key = part_of_column[column_of[variable]]; - let queries = &mut parts.get_mut(&key).expect("its column's part").queries; - if !queries.contains(&query) { - queries.push(query); - } - } - for (index, row) in milp.builder.rows.iter().enumerate() { - // A row without a variable is a constant check: kept in the part - // without a query when there is one (`None` sorts first), else in - // the first query's part. - let key = row - .expression() - .linear_coefficients() - .next() - .map(|(variable, _)| part_of_column[column_of[&variable]]) - .unwrap_or_else(|| *parts.keys().next().expect("a part")); - parts.get_mut(&key).expect("a part").rows.push(index); - } - // `None` sorts first; the part without a query goes last. - let mut parts: Vec<(Option, IndependentPart)> = parts.into_iter().collect(); - parts.sort_by_key(|(key, _)| key.unwrap_or(usize::MAX)); - parts - .into_iter() - .map(|(_, mut part)| { - part.queries.sort_unstable(); - part - }) - .collect() -} - -/// How one part's solve went, for the report. -#[derive(Debug, Clone, PartialEq, serde::Serialize)] -pub(crate) struct PartSolve { - pub(crate) queries: usize, - pub(crate) variables: usize, - pub(crate) binaries: usize, - pub(crate) rows: usize, - pub(crate) solve_ms: u64, - /// HiGHS's dual bound on the part's objective (cost per second). - pub(crate) dual_bound_per_second: f64, -} - -/// One part's model ready for HiGHS: its variables, rows and objective -/// terms, renamed to its own columns when the model has several parts. -struct PartModel { - variables: ProblemVariables, - rows: Vec, - objective_terms: Vec<(Variable, f64)>, - /// Per column of the part, the column in the whole model. - columns: Vec, - queries: usize, -} - -/// A part solved to proven optimality: its columns' values, and the HiGHS -/// objective and dual bound (cost per second). -struct SolvedPart { - columns: Vec, - objective_per_second: f64, - dual_bound_per_second: f64, - report: PartSolve, -} - -/// Solves `milp` with HiGHS, one solve per independent part -/// ([`independent_parts`]) when `split_into_parts`, each part from the warm -/// start and run to proven optimality (no time limit). -pub(crate) fn solve_selection_milp( - milp: SelectionMilp<'_>, - build_ms: u64, -) -> Result { - let size = milp.size(); - let highs_threads = milp.highs_threads; - let warm_started = Instant::now(); - let completion = super::completion::Completion::new(&milp); - let start = warm_start::warm_start(&milp, &completion); - let warm_start_ms = warm_started.elapsed().as_millis() as u64; - let warm_start_cost_per_second = start.as_ref().map(|start| start.cost_per_second); - #[cfg(feature = "lp-diagnosis")] - let diagnosis = { - let column_of: HashMap = milp - .builder - .variables - .iter_variables_with_def() - .enumerate() - .map(|(column, (variable, _))| (variable, column)) - .collect(); - let start_columns = start.as_ref().map(|start| { - let mut columns = vec![0.0; column_of.len()]; - for &(variable, value) in &start.values { - columns[column_of[&variable]] = value; - } - columns - }); - super::lp_diagnosis::Diagnosis::run(&milp, start_columns.as_deref()) - .map(|diagnosis| (diagnosis, start_columns)) - }; - let parts = if milp.split_into_parts { - independent_parts(&milp) - } else { - Vec::new() - }; - let SelectionMilp { - builder, - choice, - selected, - slot_links, - request_cost_per_second, - objective_terms, - .. - } = milp; - let ModelBuilder { - variables, rows, .. - } = builder; - let column_of: HashMap = variables - .iter_variables_with_def() - .enumerate() - .map(|(column, (variable, _))| (variable, column)) - .collect(); - let column_count = column_of.len(); - let start_columns: Option> = start.map(|start| { - let mut columns = vec![0.0; column_count]; - for (variable, value) in start.values { - columns[column_of[&variable]] = value; - } - columns - }); - let models: Vec = if parts.len() <= 1 { - vec![PartModel { - variables, - rows, - objective_terms, - columns: (0..column_count).collect(), - queries: parts.first().map_or(0, |part| part.queries.len()), - }] - } else { - let definitions: Vec<_> = variables - .iter_variables_with_def() - .map(|(_, definition)| definition.clone()) - .collect(); - let mut part_of_column = vec![0; column_count]; - let mut renamed: Vec> = vec![None; column_count]; - let mut models: Vec = parts - .iter() - .enumerate() - .map(|(index, part)| { - let mut variables = ProblemVariables::new(); - for &column in &part.columns { - part_of_column[column] = index; - renamed[column] = Some(variables.add(definitions[column].clone())); - } - PartModel { - variables, - rows: Vec::with_capacity(part.rows.len()), - objective_terms: Vec::new(), - columns: part.columns.clone(), - queries: part.queries.len(), - } - }) - .collect(); - let rename = |variable: Variable| renamed[column_of[&variable]].expect("in a part"); - for (index, part) in parts.iter().enumerate() { - for &row in &part.rows { - let row = &rows[row]; - let mut expression = Expression::from(row.expression().constant()); - for (variable, coefficient) in row.expression().linear_coefficients() { - expression.add_mul(coefficient, rename(variable)); - } - models[index].rows.push(if row.is_equality() { - constraint!(expression == 0.0) - } else { - constraint!(expression <= 0.0) - }); - } - } - for (variable, coefficient) in objective_terms { - let column = column_of[&variable]; - models[part_of_column[column]] - .objective_terms - .push((rename(variable), coefficient)); - } - models - }; - let mut columns = vec![0.0; column_count]; - let mut solved_parts: Vec = Vec::with_capacity(models.len()); - let started = Instant::now(); - for model in models { - let start = start_columns.as_ref().map(|start| { - model - .columns - .iter() - .map(|&column| start[column]) - .collect::>() - }); - let part_columns = model.columns.clone(); - let solved = solve_part(model, start, highs_threads)?; - for (&column, &value) in part_columns.iter().zip(&solved.columns) { - columns[column] = value; - } - solved_parts.push(solved); - } - let solve_ms = started.elapsed().as_millis() as u64; - let highs_objective: f64 = solved_parts - .iter() - .map(|part| part.objective_per_second) - .sum(); - let dual_bound: f64 = solved_parts - .iter() - .map(|part| part.dual_bound_per_second) - .sum(); - // The objective of the integral plan: the incumbent's binaries, - // rounded, with every continuous variable at its cheapest value (the - // solver's own value carries its tolerances, relative to the scaled - // objective). HiGHS's objective when the completion does not apply. - let objective_value = match completion.complete_columns(&columns) { - Ok(completed) => completed.cost_per_second, - Err(reason) => { - warm_start::debug(|| format!("completion of the incumbent failed: {reason}")); - highs_objective - } - }; - warm_start::debug(|| { - format!("objective: HiGHS {highs_objective:e}, completed {objective_value:e}") - }); - #[cfg(feature = "lp-diagnosis")] - if let Some((diagnosis, start_columns)) = diagnosis { - let incumbent = completion.complete_columns(&columns).ok(); - diagnosis.write( - start_columns.as_deref(), - incumbent - .as_ref() - .map(|completed| completed.columns.as_slice()), - objective_value, - dual_bound, - ); - } - let on = |variable: Variable| columns[column_of[&variable]] > 0.5; - Ok(SelectionMilpSolution { - cost_per_second: objective_value, - request_cost_per_second, - cost_lower_bound_per_second: dual_bound, - chosen: choice - .iter() - .map(|options| { - options - .iter() - .filter(|(_, &variable)| on(variable)) - .map(|(&candidate, _)| candidate) - .collect() - }) - .collect(), - selected: selected - .iter() - .filter(|(_, &variable)| on(variable)) - .map(|(&candidate, _)| candidate) - .collect(), - slot_reads: slot_links - .iter() - .filter_map(|(&key, links)| { - links - .iter() - .find(|(_, link)| on(*link)) - .map(|&(slot, _)| (key, slot)) - }) - .collect(), - size, - parts: solved_parts.into_iter().map(|part| part.report).collect(), - build_ms, - warm_start_ms, - warm_start_cost_per_second, - solve_ms, - }) -} - -/// Solves one part once with HiGHS to proven optimality, from `start` (its -/// columns' values) when given, on `highs_threads` threads. -fn solve_part( - model: PartModel, - start: Option>, - highs_threads: i32, -) -> Result { - let PartModel { - variables, - rows, - objective_terms, - columns: part_columns, - queries, - } = model; - let start_values: Option> = start.as_ref().map(|start| { - variables - .iter_variables_with_def() - .map(|(variable, _)| variable) - .zip(start.iter().copied()) - .collect() - }); - let scale = objective_scale(&objective_terms, start_values.as_deref()); - let objective: Expression = objective_terms - .iter() - .map(|&(variable, coefficient)| coefficient * scale * variable) - .sum(); - let binaries = variables - .iter_variables_with_def() - .filter(|(_, definition)| definition.is_integer()) - .count(); - let row_count = rows.len(); - let problem = variables.minimise(objective).using(highs).with_all(rows); - let mut model = problem - .try_into_inner() - .map_err(|error| SelectionMilpError::Solver(error.to_string()))?; - model.set_option("mip_rel_gap", 0.0); - model.set_option("mip_abs_gap", 0.0); - model.set_option("output_flag", false); - // HiGHS's MIP search is mostly serial; a thread per core costs - // gigabytes of address space on a large host (#1036 under 8 GB). - model.set_option("threads", highs_threads); - if let Some(start) = &start { - model - .try_set_solution(Some(start), None, None, None) - .map_err(|status| SelectionMilpError::Solver(format!("warm start: {status:?}")))?; - } - let started = Instant::now(); - let solved = model - .try_solve() - .map_err(|status| SelectionMilpError::Solver(format!("{status:?}")))?; - let solve_ms = started.elapsed().as_millis() as u64; - use highs::{HighsModelStatus, HighsSolutionStatus}; - // No limit is set, so HiGHS stops only at proven optimality or - // infeasibility; any other status is a solver failure. - match solved.status() { - HighsModelStatus::Optimal => {} - HighsModelStatus::Infeasible | HighsModelStatus::UnboundedOrInfeasible => { - return Err(SelectionMilpError::Infeasible) - } - other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), - } - if solved.primal_solution_status() != HighsSolutionStatus::Feasible { - return Err(SelectionMilpError::Solver( - "HiGHS reported an optimal status without a feasible solution".into(), - )); - } - // HiGHS leaves `mip_dual_bound` at 0 for a part without binaries (an - // LP): its optimal objective is its bound. - let dual_bound = if binaries > 0 { - solved - .double_info_value(c"mip_dual_bound") - .unwrap_or(f64::NAN) - } else { - solved.objective_value() - } / scale; - Ok(SolvedPart { - columns: solved.get_solution().columns().to_vec(), - objective_per_second: solved.objective_value() / scale, - dual_bound_per_second: dual_bound, - report: PartSolve { - queries, - variables: part_columns.len(), - binaries, - rows: row_count, - solve_ms, - dual_bound_per_second: dual_bound, - }, - }) +pub struct QueryId(pub usize); + +/// The baseline variables (section 3). +struct SelectionVariables { + /// $x_v$, by [`PhysicalCandidateId`]. + x: Vec, + /// $y_{u,c,i}$, on admissible edges only. + y: HashMap, + /// $z_{q,v}$, for $v \in G_q$. + z: HashMap<(QueryId, PhysicalCandidateId), Variable>, + /// $t_v$, by [`PhysicalCandidateId`]: only for candidates on a cycle of + /// admissible edges (a strongly connected component); `None` elsewhere. + t: Vec>, +} + +/// The execution-context variables (section 5), which replace $x$ and the +/// unindexed $y$ of the baseline. +struct ExecutionContextVariables { + /// $x^{\mathrm{mat}}_v$, for an `AcrossEvaluations` candidate. + x_materialized: HashMap, + /// $x_{\kappa,v}$, for a candidate with $\ell$ = None or `OneEvaluation`. + x_in_execution_context: HashMap<(ExecutionContextId, PhysicalCandidateId), Variable>, + /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. + y_in_execution_context: HashMap<(ExecutionContextId, PhysicalEdgeId), Variable>, + /// $h_v$, for an `AcrossEvaluations` producer. + retention_amount: HashMap, +} + +/// The accuracy variables of constraint (K) (section 4), all continuous. +struct AccuracyVariables { + /// $E_{c,m}$: candidate $c$'s error bound in metric $m$. + error_bound: HashMap<(PhysicalCandidateId, AccuracyMetric), Variable>, + /// $F_{c,i,m}$: the chosen producer's error bound in metric $m$ on input + /// $i$ of $c$. + error_bound_of_input: HashMap<(PhysicalCandidateId, usize, AccuracyMetric), Variable>, + /// $D_c$, by [`PhysicalCandidateId`]. + failure_probability: Vec, + /// $H_{c,i}$: the chosen producer's failure probability on input $i$ of $c$. + failure_probability_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, +} + +/// (A) $\sum_{u \in P_{c,i}} y_{u,c,i} = x_c$. +fn input_satisfaction_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (A)") +} + +/// (B) $y_{u,c,i} \le x_u$. +fn producer_selection_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (B)") +} + +/// (C) $\sum_{v \in G_q} z_{q,v} = 1$; (D) $z_{q,v} \le x_v$. +fn query_coverage_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraints (C) and (D)") +} + +/// (E) $t_u + 1 \le t_c + n(1 - y_{u,c,i})$, only on edges inside a strongly +/// connected component of the admissible edges, $n$ its size. +fn acyclicity_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (E)") +} + +/// (F) $x_v \le \sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v}$. +fn utilization_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (F)") +} + +/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$: for a +/// time-indexed producer (raw data, a chain, results stored per evaluation +/// time) the reading path's offset plus range, a constant per edge, so (R) +/// is linear. Query latency is ignored: a result counts as read when its +/// evaluation starts. +fn retention_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &ExecutionContextVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 7: constraint (R)") +} + +/// (K): $E_{c,m}$ per candidate and metric and $D_c$ per candidate, big-M on +/// selected edges, the metric rule on the selected path, and the root +/// checks $E_{v,m_q} \le \varepsilon_q + M_q(1 - z_{q,v})$, +/// $D_v \le \delta_q + (1 - z_{q,v})$. +fn accuracy_constraints( + physical: &PhysicalASAPMemoCandidates, + selection: &SelectionVariables, + accuracy: &AccuracyVariables, +) -> Vec { + let _ = (physical, selection, accuracy); + todo!("#1207 PR 8: constraint (K)") } - -/// The factor the objective is multiplied by for HiGHS: the warm start's -/// cost becomes about 1,000, its largest coefficient at most 10⁷; without -/// a warm start cost, the largest coefficient becomes 1,000. -pub(crate) fn objective_scale(terms: &[(Variable, f64)], start: Option<&[(Variable, f64)]>) -> f64 { - let largest = terms - .iter() - .map(|(_, coefficient)| coefficient.abs()) - .fold(0.0, f64::max); - if !(largest > 0.0 && largest.is_finite()) { - return 1.0; - } - let started: HashMap = start.into_iter().flatten().copied().collect(); - let start_cost: f64 = terms - .iter() - .map(|(variable, coefficient)| coefficient * started.get(variable).copied().unwrap_or(0.0)) - .sum(); - if start_cost > 0.0 && start_cost.is_finite() { - (1_000.0 / start_cost).min(1e7 / largest) - } else { - 1_000.0 / largest - } -} - -/// Whether this thread builds the big-M bounds as before #1095 step 1 -/// (tests only: to compare the tightened bounds with the looser ones). -#[cfg(test)] -pub(crate) fn tests_loose_big_m() -> bool { - tests::LOOSE_BIG_M.with(|loose| loose.get()) -} - -#[cfg(not(test))] -pub(crate) fn tests_loose_big_m() -> bool { - false -} - -#[cfg(test)] -pub(crate) mod tests; diff --git a/crates/plan-selection/src/milp/model/tests.rs b/crates/plan-selection/src/milp/model/tests.rs deleted file mode 100644 index a8ec9f4a0..000000000 --- a/crates/plan-selection/src/milp/model/tests.rs +++ /dev/null @@ -1,1229 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet}; - -use super::*; -use crate::groups::{prepare_milp_inputs, AccuracyWeight, IngestionPipeline}; -use crate::milp::decode::{decode, ChosenCandidates}; -use asap_cost::{node_map, plan_cost, PlanCost, ResourceUnitCost}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::stages::{AssignmentRealizer, DagOfSelectedCandidates, PhysicalCandidate}; -use asap_types::workload::{DataWorkload, DurationMs, RangeEvaluation}; - -// --------------------------------------------------------------------------- -// Through Stages 1 and 2 -// --------------------------------------------------------------------------- - -/// A workload through the real Stages 1 and 2, with the roots the -/// realizer rebuilds plans from. -pub(crate) struct Fixture { - pub(crate) physical: PhysicalASAPMemoCandidates, - pub(crate) roots: Vec, - pub(crate) entries: Vec, - pub(crate) data: DataWorkload, - pub(crate) query_time_results_retained: bool, -} - -/// One query of a fixture: PromQL over metric `m`, its repeat interval in -/// ms, its relative error target (`None`: exact), its latency bound and -/// its range evaluation. -#[derive(Clone, Copy)] -pub(crate) struct Query<'a> { - promql: &'a str, - interval_ms: u32, - max_error: Option, - max_latency_ms: Option, - range_evaluation: Option, -} - -pub(crate) fn query(promql: &str, interval_ms: u32) -> Query<'_> { - Query { - promql, - interval_ms, - max_error: None, - max_latency_ms: None, - range_evaluation: None, - } -} - -pub(crate) fn approximate(promql: &str, interval_ms: u32, max_error: f64) -> Query<'_> { - Query { - max_error: Some(max_error), - ..query(promql, interval_ms) - } -} - -/// `queries` through Stage 1 and Stage 2 as the facade runs them, over the -/// complete pane-width domain, metric `m` with 100 series sampled every -/// second. -pub(crate) fn through_stages(queries: &[Query<'_>], raw_data_retained: bool) -> Fixture { - through_stages_with(queries, raw_data_retained, false) -} - -/// [`through_stages`] over metric `m` declared non-negative (so relative -/// errors compose through quotients, and Pass 1 keeps DDSketch). -pub(crate) fn through_stages_non_negative(queries: &[Query<'_>]) -> Fixture { - through_stages_over(queries, &["m"], true, false, true) -} - -/// [`through_stages`], with rows arriving (100 per second) when -/// `ingesting`, so Stage 2 offers ingestion-time candidates. -pub(crate) fn through_stages_with( - queries: &[Query<'_>], - raw_data_retained: bool, - ingesting: bool, -) -> Fixture { - through_stages_over(queries, &["m"], raw_data_retained, ingesting, false) -} - -/// The fixture builder behind [`through_stages`], over the metrics -/// `metrics`, each with 100 series sampled every second. -fn through_stages_over( - queries: &[Query<'_>], - metrics: &[&str], - raw_data_retained: bool, - ingesting: bool, - values_non_negative: bool, -) -> Fixture { - use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; - use asap_logical_asap_optimization::generate_per_computation_logical_memo_candidates; - use asap_logical_asap_optimization::test_support::{entry_with, lower_promql}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{ - Predictability, QueryRecurrence, RepeatedDemand, RepetitionInterval, - }; - let target = |query: &Query<'_>| match query.max_error { - Some(error) => AccuracyTarget::max_error(error).failure_probability(0.01), - None => AccuracyTarget::exact(), - }; - let roots: Vec = queries - .iter() - .map(|query| QueryRoot::Operator(lower_promql(query.promql, target(query)))) - .collect(); - let entries: Vec = queries - .iter() - .map(|query| { - let mut entry = entry_with( - target(query), - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( - query.interval_ms, - ))), - ); - entry.predictability = Predictability::Predictable { known_at: None }; - entry.requirements.max_latency_ms = query.max_latency_ms; - entry.range_evaluation = query.range_evaluation; - entry - }) - .collect(); - let mut data = DataWorkload::sampled_every( - metrics.iter().map(|metric| metric.to_string()), - DurationMs(1_000), - ); - for metric in metrics { - let source = data.sources.get_mut(*metric).expect("declared"); - source.series_count = Some(100); - source.values_non_negative = values_non_negative; - } - if ingesting { - data.arrival = asap_types::workload::DataArrival::Ingesting { - rows_per_second: 100.0, - }; - } - let targets: Vec = queries.iter().map(target).collect(); - let pass1 = generate_per_computation_logical_memo_candidates( - asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - &targets, - &asap_accuracy::BuiltinAccuracyModel, - &asap_accuracy::NoAccuracyEvidence, - &data, - None, - ) - .unwrap(); - let logical = generate_logical_asap_memo_candidates(pass1, &entries).unwrap(); - let physical = asap_physical_asap_optimization::generate_physical_asap_memo_candidates( - &logical, - &entries, - &data, - false, - raw_data_retained, - ) - .unwrap(); - Fixture { - physical, - roots, - entries, - data, - query_time_results_retained: false, - } -} - -/// What one checked solve gives. -pub(crate) struct Checked { - pub(crate) solution: SelectionMilpSolution, - pub(crate) plan: PlanCost, - pub(crate) dag: DagOfSelectedCandidates, -} - -/// `fixture`'s MILP under `cost_inputs`, with `restrict` adding rows first: -/// solved, decoded, realized by Stage 2 and costed by -/// `asap_cost::plan_cost`. The objective plus the request fees must equal -/// `plan_cost`'s cost. -pub(crate) fn solve_and_check( - fixture: &Fixture, - cost_inputs: &CostInputs, - restrict: impl FnOnce(&mut SelectionMilp<'_>), -) -> Result { - solve_and_check_by(fixture, cost_inputs, restrict, true) -} - -/// [`solve_and_check`] with the independent parts solved apart when -/// `split_into_parts`. -fn solve_and_check_by( - fixture: &Fixture, - cost_inputs: &CostInputs, - restrict: impl FnOnce(&mut SelectionMilp<'_>), - split_into_parts: bool, -) -> Result { - let inputs = prepare_milp_inputs( - &fixture.physical, - &fixture.entries, - &fixture.data, - cost_inputs, - ); - let mut milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - cost_inputs, - SelectionMilpOptions { - split_into_parts, - ..SelectionMilpOptions::default() - }, - )?; - assert!(milp.inexact.is_empty(), "inexact terms: {:?}", milp.inexact); - restrict(&mut milp); - let solution = solve_selection_milp(milp, 0)?; - let dag = decode( - &fixture.physical, - &inputs, - &ChosenCandidates { - chosen: solution.chosen.clone(), - selected: solution.selected.clone(), - slot_reads: solution.slot_reads.clone(), - }, - ) - .expect("the solution decodes"); - dag.validate(&fixture.physical) - .expect("the decoded selection is valid"); - let realizer = asap_physical_asap_optimization::Stage2AssignmentRealizer { - original_roots: &fixture.roots, - entries: &fixture.entries, - data: &fixture.data, - query_time_results_retained: fixture.query_time_results_retained, - }; - let realized = realizer - .realize(&fixture.physical, &dag) - .expect("the realizer accepts the decoded plan"); - let plan_dag = &realized.plan.dag; - let recurrence: Vec<_> = fixture - .entries - .iter() - .map(|entry| entry.recurrence.clone()) - .collect(); - let ranges: Vec<_> = fixture - .entries - .iter() - .map(|entry| entry.range_evaluation) - .collect(); - let (plan, _) = plan_cost( - plan_dag, - &node_map(plan_dag), - &asap_cost::kept_tumbling_window_chains(plan_dag), - &recurrence, - &ranges, - &fixture.data, - cost_inputs, - cost_inputs.frozen_calibration_factor.unwrap_or(1.0), - ) - .expect("plan_cost costs the plan"); - let expected = plan.cost_per_second.expect("cost model 1") - solution.request_cost_per_second; - let tolerance = 1e-6 * expected.abs().max(1e-12); - assert!( - (solution.cost_per_second - expected).abs() <= tolerance, - "MILP objective {:e} != plan_cost {:e} (without {:e} of request fees); per node: {:#?}", - solution.cost_per_second, - expected, - solution.request_cost_per_second, - plan.per_node - ); - Ok(Checked { - solution, - plan, - dag, - }) -} - -pub(crate) fn cost_inputs() -> CostInputs { - CostInputs { - frozen_calibration_factor: Some(1.0), - ..CostInputs::default() - } -} - -/// The `aws-fargate-us-east-1` cost model of the memo examples. -fn aws_fargate_cost_inputs() -> CostInputs { - let defaults = cost_inputs(); - CostInputs { - cost_model: asap_cost::CostModel::UsageOverTime(ResourceUnitCost::aws_fargate_us_east_1( - defaults.resource_speed, - )), - ..defaults - } -} - -/// The candidates of `milp` that satisfy `wanted`. -fn candidates_where( - milp: &SelectionMilp<'_>, - wanted: impl Fn(&PhysicalCandidate, &OperatorNode) -> bool, -) -> Vec { - (0..milp.physical.candidates.len()) - .map(PhysicalCandidateId) - .filter(|&candidate| { - wanted( - &milp.physical.candidates[candidate.0], - milp.physical.physical_node(candidate), - ) - }) - .collect() -} - -/// Adds `Σ x[m] ≥ 1` over the shareable `candidates`: one is selected. -fn require_one_of(milp: &mut SelectionMilp<'_>, candidates: &[PhysicalCandidateId]) { - let sum: Expression = candidates - .iter() - .filter_map(|candidate| milp.selected.get(candidate).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("test", constraint!(sum >= 1.0)); -} - -/// Adds `x[m] = 0` and `p[ctx, c] = 0` everywhere for `candidates`. -fn forbid(milp: &mut SelectionMilp<'_>, candidates: &[PhysicalCandidateId]) { - for candidate in candidates { - if let Some(&selected) = milp.selected.get(candidate) { - milp.builder.row("test", constraint!(selected <= 0.0)); - } - for context in 0..milp.choice.len() { - if let Some(&choice) = milp.choice[context].get(candidate) { - milp.builder.row("test", constraint!(choice <= 0.0)); - } - } - } -} - -/// The candidates that are pane chains or run at ingestion time. -fn pane_chains_and_ingestion(milp: &SelectionMilp<'_>) -> Vec { - candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::IngestionTime - || matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) -} - -/// How many nodes of `dag` run `candidate`. -fn runs_of(dag: &DagOfSelectedCandidates, candidate: PhysicalCandidateId) -> usize { - dag.candidate_of_node - .iter() - .filter(|&&c| c == candidate) - .count() -} - -/// `sum(m)` repeating every minute and `sum(m) + max(m)` every 30 s: the -/// second query does more work, so its evaluations take longer. -pub(crate) fn shared_sum() -> Fixture { - through_stages( - &[query("sum(m)", 60_000), query("sum(m) + max(m)", 30_000)], - true, - ) -} - -/// `sum_over_time(m[5m])` and `sum_over_time(m[1h])` over raw data. -pub(crate) fn two_windows() -> Fixture { - through_stages( - &[ - query("sum_over_time(m[5m])", 60_000), - query("sum_over_time(m[1h])", 60_000), - ], - true, - ) -} - -// --------------------------------------------------------------------------- -// Cost terms equal `plan_cost` (§3) -// --------------------------------------------------------------------------- - -// The optimum's objective equals `plan_cost` of the plan it decodes to. -#[test] -fn the_optimum_costs_what_plan_cost_says() { - for costs in [cost_inputs(), aws_fargate_cost_inputs()] { - solve_and_check(&shared_sum(), &costs, |_| {}).unwrap(); - } -} - -// Held memory and schedule groups: a `MatAtQuery(Memory)` result both -// queries read (every 60 s and every 30 s) is held until the slower reader -// finishes, once per execution at the schedule groups' rates. -#[test] -fn held_memory_of_a_result_readers_of_different_latency_share() { - let fixture = shared_sum(); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let shared: Vec = candidates_where(milp, |candidate, _| { - candidate.materialization == Materialization::MatAtQuery(Medium::Memory) - }) - .into_iter() - .filter(|&candidate| milp.queries_reaching(candidate).len() == 2) - .collect(); - assert!(!shared.is_empty()); - require_one_of(milp, &shared); - }) - .unwrap(); - let latencies: Vec = checked - .plan - .usage - .latency_per_query - .iter() - .map(|latency| latency.total_ms()) - .collect(); - assert!(latencies[0] != latencies[1], "{latencies:?}"); - assert!(checked - .plan - .per_node - .values() - .any(|node| node.detail.contains("holds"))); -} - -// Raw retention: with one scan per window, the source keeps the longer -// window's raw data once. -#[test] -fn raw_retention_keeps_the_longest_window() { - let checked = solve_and_check(&two_windows(), &cost_inputs(), |milp| { - let excluded = pane_chains_and_ingestion(milp); - forbid(milp, &excluded); - }) - .unwrap(); - assert!(checked - .plan - .per_node - .values() - .any(|node| node.detail.contains("of raw data on disk"))); -} - -// A scan read through two offsets in one query evaluation: the scan run -// reads the union of the two windows, and the MILP charges exactly that -// union (`δ` over the window kinds read), as `plan_cost` does. -#[test] -fn a_scan_read_through_two_offsets_is_costed_at_their_union() { - let fixture = through_stages( - &[query( - "sum_over_time(m[5m]) + sum_over_time(m[5m] offset 1h)", - 60_000, - )], - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let excluded = pane_chains_and_ingestion(milp); - forbid(milp, &excluded); - }) - .unwrap(); - assert!( - checked - .solution - .size - .families - .contains_key("δ (scan window sets)"), - "{:?}", - checked.solution.size.families.keys().collect::>() - ); -} - -// A kept pane chain whose readers have different range-evaluation spans: -// it keeps the panes of the longest span, charged by retention levels. -#[test] -fn a_kept_pane_chain_keeps_the_longest_readers_span() { - let span = |span_ms| { - Some(RangeEvaluation { - step_ms: 60_000, - span_ms, - }) - }; - let fixture = through_stages_with( - &[ - Query { - range_evaluation: span(600_000), - ..query("sum_over_time(m[1h])", 60_000) - }, - Query { - range_evaluation: span(3_600_000), - ..query("sum_over_time(m[1h])", 60_000) - }, - ], - false, - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let chains = candidates_where(milp, |candidate, node| { - matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) && matches!( - candidate.materialization, - Materialization::MatAtIngestion(_) - ) - }); - require_one_of(milp, &chains); - }) - .unwrap(); - assert!( - checked - .solution - .size - .families - .contains_key("π (pane retention levels)"), - "{:?}", - checked.solution.size.families.keys().collect::>() - ); -} - -// A `NotMaterialized` input of a `MatAtQuery` candidate both queries read -// (`rate(m[5m])` under `sum` and `max`, one schedule group) runs once, in -// the candidate's context, and is costed as `plan_cost` costs it. -#[test] -fn a_shared_results_not_materialized_input_runs_once() { - let fixture = through_stages( - &[ - query("sum(rate(m[5m]))", 60_000), - query("max(rate(m[5m]))", 60_000), - ], - true, - ); - let mut shared = Vec::new(); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - shared = candidates_where(milp, |candidate, _| { - matches!(candidate.materialization, Materialization::MatAtQuery(_)) - }) - .into_iter() - .filter(|&m| milp.queries_reaching(m).len() == 2) - .filter(|&m| { - let context = milp.context_of(m).expect("a context"); - milp.choice[context] - .keys() - .any(|&c| milp.computes(context, c)) - }) - .collect(); - assert!(!shared.is_empty(), "a shared result over a computed input"); - require_one_of(milp, &shared); - }) - .unwrap(); - let m = *checked - .solution - .selected - .iter() - .find(|m| shared.contains(m)) - .expect("selected"); - assert_eq!(runs_of(&checked.dag, m), 1); - let node = checked - .dag - .candidate_of_node - .iter() - .position(|&c| c == m) - .expect("a node"); - for edge in checked.dag.edges.iter().filter(|e| e.consumer.0 == node) { - let input = checked.dag.candidate_of_node[edge.producer.0]; - let readers = checked - .dag - .edges - .iter() - .filter(|e| e.producer == edge.producer) - .count(); - assert_eq!(readers, 1, "m's input {input:?} is read by m alone"); - } -} - -// One ingestion scan read by sketches of two configurations (two quantile -// targets) runs once in its `I_r` and is costed once. -#[test] -fn one_ingestion_scan_shared_by_two_sketch_configurations_is_charged_once() { - let fixture = through_stages_with( - &[ - approximate("quantile_over_time(0.5, m[5m])", 60_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - true, - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let sketches = candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::IngestionTime - && candidate.materialization != Materialization::NotMaterialized - && matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryAgg { .. }) - | Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }); - let sum: Expression = sketches - .iter() - .filter_map(|c| milp.selected.get(c).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("test", constraint!(sum >= 2.0)); - // Every query-time scan is excluded: both read ingestion state. - let scans = candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::QueryTime && node.operator.kind_name() == "Scan" - }); - forbid(milp, &scans); - }) - .unwrap(); - let ingestion_scans: Vec = (0..checked.dag.candidate_of_node.len()) - .filter(|&node| { - let candidate = checked.dag.candidate_of_node[node]; - fixture.physical.candidates[candidate.0].timing == ExecutionTiming::IngestionTime - && fixture - .physical - .physical_node(candidate) - .operator - .kind_name() - == "Scan" - }) - .collect(); - assert_eq!(ingestion_scans.len(), 1, "{:?}", checked.dag); -} - -/// Solves `queries` (ingesting) with two ingestion-time sketch states -/// selected, every query-time scan forbidden and every ingestion-time state -/// whose refresh window is not one of `widths_ms` (phase 0) forbidden; -/// returns the checked solve (objective = `plan_cost`) and its -/// ingestion-time scan runs, each with the refresh windows of the states -/// reading it. -fn ingestion_scans_under_refresh_windows( - queries: &[Query<'_>], - widths_ms: &[u64], -) -> (Checked, Vec>) { - use asap_types::ir::materialized_view::RefreshWindow; - let fixture = through_stages_with(queries, true, true); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let states = candidates_where(milp, |candidate, _| { - candidate.timing == ExecutionTiming::IngestionTime - && candidate.materialization != Materialization::NotMaterialized - }); - let window_of = |state: PhysicalCandidateId| { - milp.inputs - .ingestion_context_of_shareable - .get(&state) - .map(|&context| milp.inputs.contexts[context].kind) - }; - let (allowed, other): (Vec<_>, Vec<_>) = states.iter().partition(|&&state| { - widths_ms.iter().any(|&width_ms| { - window_of(state) - == Some(ContextKind::Ingestion(IngestionPipeline::Window( - RefreshWindow { - width_ms, - phase_ms: 0, - }, - ))) - }) - }); - forbid(milp, &other); - let sketches = candidates_where(milp, |_, node| { - matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }); - let sum: Expression = allowed - .iter() - .filter(|state| sketches.contains(state)) - .filter_map(|state| milp.selected.get(state).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("test", constraint!(sum >= 2.0)); - let scans = candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::QueryTime && node.operator.kind_name() == "Scan" - }); - forbid(milp, &scans); - }) - .unwrap(); - // Per ingestion-time scan node of the decoded selection: the pane widths - // of the chains above it. - let dag = &checked.dag; - let nodes = dag.candidate_of_node.len(); - let mut readers: Vec> = vec![Vec::new(); nodes]; - for edge in &dag.edges { - readers[edge.producer.0].push(edge.consumer.0); - } - let width_above = |scan: usize| { - let mut widths = BTreeSet::new(); - let mut stack = vec![scan]; - let mut seen = BTreeSet::new(); - while let Some(node) = stack.pop() { - if !seen.insert(node) { - continue; - } - let candidate = dag.candidate_of_node[node]; - match fixture.physical.physical_node(candidate).asap() { - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => { - widths.insert(*tumbling_window_width_ms); - } - _ => stack.extend(readers[node].iter().copied()), - } - } - widths - }; - let scans = (0..nodes) - .filter(|&node| { - let candidate = dag.candidate_of_node[node]; - fixture.physical.candidates[candidate.0].timing == ExecutionTiming::IngestionTime - && fixture - .physical - .physical_node(candidate) - .operator - .kind_name() - == "Scan" - }) - .map(width_above) - .collect(); - (checked, scans) -} - -// Two ingestion-time pane chains of one refresh window (10 s panes) over -// one source run in one ingestion pipeline: the scan below them runs and -// is charged once, and the objective equals `plan_cost`. -#[test] -fn one_ingestion_scan_under_chains_of_one_refresh_window_is_charged_once() { - let (_, scans) = ingestion_scans_under_refresh_windows( - &[ - approximate("quantile_over_time(0.5, m[5m])", 60_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - &[10_000], - ); - assert_eq!(scans, vec![BTreeSet::from([10_000])]); -} - -// Chains of 10 s and 60 s panes over one source run in two ingestion -// pipelines (#1151): the scan runs and is charged once in each, and the -// objective equals `plan_cost`, which charges it once per refresh window. -#[test] -fn ingestion_scans_under_chains_of_two_refresh_windows_are_charged_twice() { - let (_, scans) = ingestion_scans_under_refresh_windows( - &[ - approximate("quantile_over_time(0.5, m[10s])", 10_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - &[10_000, 60_000], - ); - let mut scans = scans; - scans.sort(); - assert_eq!( - scans, - vec![BTreeSet::from([10_000]), BTreeSet::from([60_000])] - ); -} - -// Every producer candidate of every input reads one view window on real -// Stage 2 output (ingesting, and not), so a refresh window computed from -// first producers is the realized plan's. -#[test] -fn producers_of_an_input_read_one_view_window_through_stages() { - let fixtures = [ - through_stages_with( - &[ - approximate("quantile_over_time(0.5, m[10s])", 10_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - true, - true, - ), - through_stages_with( - &[ - query("sum by (job) (rate(m[1m]))", 30_000), - query("sum(increase(m[5m] offset 1m))", 60_000), - approximate( - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - 60_000, - 0.05, - ), - ], - true, - true, - ), - through_stages( - &[ - query("avg_over_time(m[5m])", 60_000), - query("max_over_time(m[1m])", 60_000), - ], - true, - ), - ]; - for fixture in &fixtures { - assert_eq!( - crate::groups::producer_window_disagreements_of_all(&fixture.physical), - Vec::new() - ); - } -} - -// A stateless subtree read by two state nodes of one query evaluation -// (`avg` reads a sum and a count accumulator over one `q50/q50 + q90` -// subtree) is computed once: no candidate runs twice in the plan. -#[test] -fn a_stateless_subtree_two_state_nodes_read_runs_once() { - let fixture = through_stages( - &[query( - "avg by (job) (quantile_over_time(0.5, m[5m]) / quantile_over_time(0.5, m[5m]) \ - + quantile_over_time(0.9, m[5m]))", - 60_000, - )], - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |_| {}).unwrap(); - let mut runs: BTreeMap = BTreeMap::new(); - for &candidate in &checked.dag.candidate_of_node { - *runs.entry(candidate).or_default() += 1; - } - assert!(runs.values().all(|&n| n == 1), "{runs:?}"); -} - -// --------------------------------------------------------------------------- -// Structure -// --------------------------------------------------------------------------- - -// "Used only if read": a query context may not select a shareable -// candidate no used reader there reads, so a phantom read cannot satisfy -// `MinimumQueryReaders`. With every reader of a two-reader `MatAtQuery` -// result in the second query's context forbidden, selecting it is -// infeasible. -#[test] -fn a_phantom_reader_does_not_satisfy_minimum_query_readers() { - let fixture = shared_sum(); - let costs = cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let options = SelectionMilpOptions::default(); - let mut milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - options, - ) - .unwrap(); - let constrained: Vec = fixture - .physical - .reader_constraints - .iter() - .filter_map(|constraint| match constraint { - ReaderConstraint::MinimumQueryReaders { - candidate, minimum, .. - } if *minimum >= 2 => Some(*candidate), - _ => None, - }) - .filter(|&m| milp.selected.contains_key(&m) && milp.choice[1].contains_key(&m)) - .collect(); - let m = *constrained.first().expect("a two-reader shareable result"); - let selected = milp.selected[&m]; - milp.builder.row("test", constraint!(selected >= 1.0)); - let readers: Vec = milp.readers[1][&m] - .iter() - .filter_map(|reader| match reader { - Reader::Candidate(reader, _) => Some(*reader), - _ => None, - }) - .collect(); - assert!( - !milp.readers[1][&m] - .iter() - .any(|reader| matches!(reader, Reader::Output(_))), - "the result is not an output of the second query" - ); - for reader in readers { - let choice = milp.choice[1][&reader]; - milp.builder.row("test", constraint!(choice <= 0.0)); - } - assert_eq!( - solve_selection_milp(milp, 0).err(), - Some(SelectionMilpError::Infeasible) - ); -} - -// A `NotMaterialized` query-time candidate selected in two contexts is two -// nodes; a shareable candidate two queries read is one node. -#[test] -fn runs_decode_one_node_per_context_and_one_per_shareable_candidate() { - let fixture = through_stages( - &[query("sum(m)", 60_000), query("sum(m) * 2", 60_000)], - true, - ); - let unshared = solve_and_check(&fixture, &cost_inputs(), |milp| { - let shared = candidates_where(milp, |candidate, _| { - candidate.timing == ExecutionTiming::QueryTime - && candidate.materialization != Materialization::NotMaterialized - }); - forbid(milp, &shared); - let ingestion = candidates_where(milp, |candidate, _| { - candidate.timing == ExecutionTiming::IngestionTime - }); - forbid(milp, &ingestion); - }) - .unwrap(); - let mut runs: BTreeMap = BTreeMap::new(); - for &candidate in &unshared.dag.candidate_of_node { - *runs.entry(candidate).or_default() += 1; - } - assert!(runs.values().any(|&n| n == 2), "{runs:?}"); - let mut shared_result = Vec::new(); - let shared = solve_and_check(&fixture, &cost_inputs(), |milp| { - shared_result = candidates_where(milp, |candidate, _| { - candidate.materialization == Materialization::MatAtQuery(Medium::Memory) - }) - .into_iter() - .filter(|&m| milp.queries_reaching(m).len() == 2) - .collect(); - require_one_of(milp, &shared_result); - }) - .unwrap(); - let m = *shared - .solution - .selected - .iter() - .find(|m| shared_result.contains(m)) - .expect("selected"); - assert_eq!(runs_of(&shared.dag, m), 1); -} - -// A query over its latency bound: with the bound below the fastest plan's -// latency there is no plan; at the optimum's latency, the optimum stays. -#[test] -fn a_plan_over_a_latency_bound_is_infeasible() { - let fixture = shared_sum(); - let optimum = solve_and_check(&fixture, &cost_inputs(), |_| {}).unwrap(); - let latency = optimum.plan.usage.latency_per_query[1].total_ms(); - let mut bounded = shared_sum(); - bounded.entries[1].requirements.max_latency_ms = Some(latency * (1.0 + 1e-9)); - let at_bound = solve_and_check(&bounded, &cost_inputs(), |_| {}).unwrap(); - assert!(at_bound.plan.usage.latency_per_query[1].total_ms() <= latency * (1.0 + 1e-9)); - let mut tight = shared_sum(); - tight.entries[1].requirements.max_latency_ms = Some(1e-9); - assert!(matches!( - solve_and_check(&tight, &cost_inputs(), |_| {}).err(), - Some(SelectionMilpError::Infeasible) - )); -} - -// Cost models 2 and 3 are not encoded. -#[test] -fn cost_models_2_and_3_are_unsupported() { - use asap_cost::{ClusterLimits, ClusterObjective, CostModel}; - let fixture = shared_sum(); - for model in [ - CostModel::ProvisionedCapacity(ResourceUnitCost::illustrative()), - CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective: ClusterObjective::LowestLatency, - }), - ] { - let costs = CostInputs { - cost_model: model, - ..cost_inputs() - }; - let inputs = - prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let options = SelectionMilpOptions::default(); - assert!(matches!( - build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - options - ) - .err(), - Some(SelectionMilpError::UnsupportedCostModel(_)) - )); - } -} - -// Accuracy rows on `q0.5 / q0.9` over metric `m` declared non-negative, -// weights from the composition: a candidate unusable for the query (a -// DDSketch denominator sized for the whole target) is never depended on, -// and two at `parts = 2` are depended on together unless their pair is -// excluded. (Pass 1 already prunes KLL, whose rank error composes no -// guarantee through the quotient.) -#[test] -fn accuracy_rows_follow_the_composition_on_a_ratio() { - let fixture = through_stages_non_negative(&[approximate( - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - 60_000, - 0.01, - )]); - let costs = cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let of = |wanted: &dyn Fn(&AccuracyWeight, usize) -> bool| -> Vec { - inputs.accuracy_weights[0] - .iter() - .filter(|(class, weight)| { - wanted(weight, inputs.candidate_data[class.0].accuracy_budget_parts) - }) - .map(|(class, _)| *class) - .collect() - }; - let unusable = of(&|weight, _| matches!(weight, AccuracyWeight::Unusable(_))); - let usable_at_two = of(&|weight, parts| *weight == AccuracyWeight::One && parts == 2); - assert!(!unusable.is_empty() && usable_at_two.len() >= 2); - let solve = |inputs: &MilpInputs, restrict: &dyn Fn(&mut SelectionMilp<'_>)| { - let mut milp = build_selection_milp( - &fixture.physical, - inputs, - &fixture.entries, - &costs, - SelectionMilpOptions::default(), - ) - .unwrap(); - restrict(&mut milp); - solve_selection_milp(milp, 0).map(|_| ()) - }; - // Each candidate's runs: `x[m]`, or `p` in a context computing it. - let runs_of = |milp: &SelectionMilp<'_>, candidates: &[PhysicalCandidateId]| -> Expression { - let mut sum = Expression::from(0.0); - for &candidate in candidates { - if let Some(&selected) = milp.selected.get(&candidate) { - sum += selected; - } - for context in 0..milp.choice.len() { - if milp.computes(context, candidate) { - if let Some(&choice) = milp.choice[context].get(&candidate) { - sum += choice; - } - } - } - } - sum - }; - assert_eq!( - solve(&inputs, &|milp| { - let sum = runs_of(milp, &unusable); - milp.builder.row("test", constraint!(sum >= 1.0)); - }) - .err(), - Some(SelectionMilpError::Infeasible) - ); - let two_of = |milp: &mut SelectionMilp<'_>| { - let sum = runs_of(milp, &usable_at_two); - milp.builder.row("test", constraint!(sum >= 2.0)); - }; - assert!(solve(&inputs, &two_of).is_ok()); - let mut excluded = inputs.clone(); - for (index, &first) in usable_at_two.iter().enumerate() { - for &second in &usable_at_two[index + 1..] { - excluded.accuracy_excluded_pairs[0].push((first, second)); - } - } - assert_eq!( - solve(&excluded, &two_of).err(), - Some(SelectionMilpError::Infeasible) - ); -} - -// The warm start is a feasible plan whose completion's objective equals -// what `plan_cost` says of it: fixing every binary to it, the solve -// returns the warm start's cost. -#[test] -fn the_warm_start_is_a_feasible_plan_costed_exactly() { - for fixture in [shared_sum(), two_windows()] { - let mut start_cost = None; - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let completion = crate::milp::completion::Completion::new(milp); - let start = - crate::milp::warm_start::warm_start(milp, &completion).expect("a warm start"); - start_cost = Some(start.cost_per_second); - let values: HashMap = start.values.into_iter().collect(); - let mut fixed = Vec::new(); - for choices in &milp.choice { - fixed.extend(choices.values().copied()); - } - fixed.extend(milp.selected.values().copied()); - for variable in fixed { - let value = values.get(&variable).copied().unwrap_or(0.0); - milp.builder.row("test", constraint!(variable == value)); - } - }) - .unwrap(); - let start_cost = start_cost.expect("costed"); - let tolerance = 1e-6 * start_cost.abs().max(1e-12); - assert!( - (checked.solution.cost_per_second - start_cost).abs() <= tolerance, - "warm start {start_cost:e} != plan_cost {:e}", - checked.solution.cost_per_second - ); - } -} - -// RangeHeavy templates 7 and 4: the widened scan puts a `Project` in the -// memo group of the scan it reads. The group's roots are split by -// self-read depth, so the plan is acyclic, valid, accepted by the realizer -// and costed exactly. -#[test] -fn a_widened_scan_reading_its_own_memo_group_plans_exactly() { - let fixture = through_stages( - &[ - approximate("sum by (job) (sum_over_time(m[1h]))", 60_000, 0.05), - approximate("sum_over_time(m[1h])", 60_000, 0.05), - ], - true, - ); - let reads_own_group = (0..fixture.physical.candidates.len()).any(|candidate| { - let group = fixture - .physical - .memo_group_of(PhysicalCandidateId(candidate)); - fixture.physical.candidates[candidate] - .producer_candidates_per_input - .iter() - .any(|producers| { - producers - .iter() - .any(|p| p.0 != candidate && fixture.physical.memo_group_of(*p) == group) - }) - }); - assert!( - reads_own_group, - "the fixture has a root reading its own group" - ); - for costs in [cost_inputs(), aws_fargate_cost_inputs()] { - solve_and_check(&fixture, &costs, |_| {}).unwrap(); - } -} - -// #1095 §5 (3a): queries over two metrics share no candidate, run of an `I_r` or -// source, so the model splits into one part per metric; solving the parts -// apart gives the plan and cost of one solve of the whole model. -#[test] -fn independent_query_sets_solve_apart_to_the_same_plan() { - let queries = [ - query("sum_over_time(m[5m])", 60_000), - query("sum_over_time(m[1h])", 60_000), - query("sum(n) + max(n)", 30_000), - ]; - let fixture = through_stages_over(&queries, &["m", "n"], true, false, false); - let costs = cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let options = SelectionMilpOptions::default(); - let milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - options, - ) - .unwrap(); - let parts = independent_parts(&milp); - let queries_per_part: Vec> = parts.iter().map(|part| part.queries.clone()).collect(); - assert_eq!(queries_per_part, vec![vec![0, 1], vec![2]]); - let columns: usize = parts.iter().map(|part| part.columns.len()).sum(); - let rows: usize = parts.iter().map(|part| part.rows.len()).sum(); - assert_eq!(columns, milp.builder.variables.len()); - assert_eq!(rows, milp.builder.rows.len()); - - let apart = solve_and_check_by(&fixture, &costs, |_| {}, true).unwrap(); - let whole = solve_and_check_by(&fixture, &costs, |_| {}, false).unwrap(); - assert_eq!(apart.solution.parts.len(), 2); - assert_eq!(whole.solution.parts.len(), 1); - let tolerance = 1e-9 * whole.solution.cost_per_second.abs(); - assert!( - (apart.solution.cost_per_second - whole.solution.cost_per_second).abs() <= tolerance, - "apart {:e} != whole {:e}", - apart.solution.cost_per_second, - whole.solution.cost_per_second - ); - assert_eq!(apart.dag.candidate_of_node, whole.dag.candidate_of_node); -} - -thread_local! { - /// Set by a test to build the big-M bounds as before #1095 step 1 - /// ([`super::tests_loose_big_m`]). - pub(crate) static LOOSE_BIG_M: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - -/// Per row of family `family` in `fixture`'s model (built with the loose -/// big-M bounds when `loose`), the largest coefficient on an `x` variable. -fn big_m_per_row(fixture: &Fixture, family: &str, loose: bool) -> Vec { - LOOSE_BIG_M.with(|cell| cell.set(loose)); - let costs = aws_fargate_cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - SelectionMilpOptions::default(), - ) - .unwrap(); - LOOSE_BIG_M.with(|cell| cell.set(false)); - let selected: std::collections::HashSet = milp.selected.values().copied().collect(); - milp.builder - .rows - .iter() - .zip(&milp.builder.row_families) - .filter(|(_, &f)| f == family) - .map(|(row, _)| { - row.expression() - .linear_coefficients() - .filter(|(variable, _)| selected.contains(variable)) - .map(|(_, coefficient)| coefficient.abs()) - .fold(0.0, f64::max) - }) - .collect() -} - -// #1095 step 1: `Ē[m]` in `V ≥ E − Ē (x[m] − z)` is the sum over the -// context's slots of its costliest run. With a rollup read every 20 s and -// every 30 s, a shared context has slots with several runs, so the bound is -// below the old sum over every run; the plan and its cost are unchanged. -#[test] -fn a_tighter_context_execution_bound_keeps_the_plan() { - let fixture = through_stages( - &[ - query("sum by (job) (sum_over_time(m[10m]))", 20_000), - query("sum(sum_over_time(m[10m]))", 30_000), - ], - true, - ); - let family = "V (context execution at a rate component)"; - let tight = big_m_per_row(&fixture, family, false); - let loose = big_m_per_row(&fixture, family, true); - assert_eq!(tight.len(), loose.len()); - assert!(tight - .iter() - .zip(&loose) - .all(|(t, l)| *t <= l * (1.0 + 1e-12))); - assert!(tight.iter().zip(&loose).any(|(t, l)| *t < l * (1.0 - 1e-9))); - - let tightened = solve_and_check(&fixture, &aws_fargate_cost_inputs(), |_| {}).unwrap(); - LOOSE_BIG_M.with(|cell| cell.set(true)); - let looser = solve_and_check(&fixture, &aws_fargate_cost_inputs(), |_| {}); - LOOSE_BIG_M.with(|cell| cell.set(false)); - let looser = looser.unwrap(); - let tolerance = 1e-9 * looser.solution.cost_per_second.abs(); - assert!( - (tightened.solution.cost_per_second - looser.solution.cost_per_second).abs() <= tolerance - ); - assert_eq!( - tightened.dag.candidate_of_node, - looser.dag.candidate_of_node - ); -} diff --git a/crates/plan-selection/src/milp/warm_start.rs b/crates/plan-selection/src/milp/warm_start.rs deleted file mode 100644 index b7de6ae37..000000000 --- a/crates/plan-selection/src/milp/warm_start.rs +++ /dev/null @@ -1,780 +0,0 @@ -//! A feasible starting plan for the group selection MILP (#1095 §2.0, -//! "Solving strategy" step 1), so HiGHS starts with an incumbent. -//! -//! **Each query's cheapest plan.** Queries are planned one after another -//! into one plan. A query's plan is derived from its output references -//! down: in each context, each slot an input needs takes the candidate -//! with the cheapest estimated subtree (its usage at the context's rate -//! plus its inputs' cheapest subtrees), unless the context already chose a -//! candidate of that slot, which every reader must then accept (one -//! candidate per slot per context); a dead end backtracks to the next -//! candidate. A shareable candidate's context is derived once, when it is -//! first selected; a candidate of an `I_r` is chosen in the `I_r` of the -//! reading context's refresh window. -//! -//! **Repairs per query.** If the query's plan depends on more approximate -//! values than one of them is sized for, on a pair excluded together, or -//! on a candidate unusable for it, the query is planned again with exact -//! candidates only; if its estimated latency is over its bound, it is -//! planned again by latency alone. -//! -//! **Greedy sharing.** A second plan prices what the queries planned -//! before already selected (a shareable candidate, a candidate of an `I_r`) at -//! zero, so later queries read it. Both plans, in workload order and -//! reversed, are costed exactly by the MILP's own rows and objective -//! ([`Completion`]); the cheapest feasible one is the start. -use std::collections::{BTreeMap, BTreeSet, HashMap}; - -use asap_cost::execution_demand::evaluation_rate; -use asap_types::stages::{output_references, PhysicalCandidateId, ReaderConstraint}; -use good_lp::Variable; - -use crate::groups::{AccuracyWeight, ContextKind, ResultSlot, RunKind}; - -use super::completion::Completion; -use super::cost_terms::NodeNumbers; -use super::model::{latency_bound_ms, SelectionMilp}; - -/// The most derivation steps one query's plan takes before giving up. -const MOST_STEPS_PER_QUERY: usize = 200_000; - -/// The starting plan. -pub(crate) struct StartingPlan { - /// Every variable's value. - pub(crate) values: Vec<(Variable, f64)>, - /// Its objective (cost per second without the request fees). - pub(crate) cost_per_second: f64, -} - -/// What a derivation minimizes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Objective { - Cost, - Latency, -} - -/// The candidates a query's derivation may use. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Approximate { - /// Approximate candidates usable for the query. - Usable, - /// Those sized for at least this many shares. - SizedFor(usize), - /// Exact candidates only. - None, -} - -/// A plan under construction: per context, the candidate of each slot, and -/// the selected shareable candidates. -#[derive(Debug, Clone, Default)] -struct Plan { - chosen: Vec>, - selected: BTreeSet, - /// Per multi-slot input `(context, reader, input)`, the slot it reads. - slot_reads: BTreeMap<(usize, PhysicalCandidateId, usize), ResultSlot>, -} - -/// One change to a [`Plan`], to undo. -enum Undo { - Chose(usize, ResultSlot), - Selected(PhysicalCandidateId), - SlotRead(usize, PhysicalCandidateId, usize), -} - -/// A feasible plan (module documentation), or `None` when none was found. -/// It tries a fixed number of query orders ([`MORE_ORDERS`]), so every run -/// starts from the same plan. -pub(crate) fn warm_start( - milp: &SelectionMilp<'_>, - completion: &Completion, -) -> Option { - let estimates = Estimates::new(milp); - let mut best: Option = None; - let count = milp.query_count(); - let mut orders: Vec> = vec![(0..count).collect(), (0..count).rev().collect()]; - // More orders: a fixed pseudo-random sequence of permutations, so runs - // are reproducible. - let mut state: u64 = 0x9e37_79b9_7f4a_7c15; - for _ in 0..MORE_ORDERS { - let mut order: Vec = (0..count).collect(); - for at in (1..count).rev() { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - order.swap(at, (state % (at as u64 + 1)) as usize); - } - orders.push(order); - } - // (share, amortized): alone; reading what earlier queries selected; and - // that with each shareable candidate's own cost divided among the - // queries that may reach it. - let strategies = [(false, false), (true, false), (true, true)]; - let mut tried = 0; - for order in &orders { - for &(share, amortized) in &strategies { - tried += 1; - let Some(plan) = - sequential_plan(milp, &estimates, order, share, amortized, &BTreeSet::new()) - else { - debug(|| format!("warm start: no plan (share {share}, amortized {amortized})")); - continue; - }; - let binaries = binaries_of(milp, &plan); - let completed = completion.complete(&binaries); - if let Err(reason) = &completed { - debug(|| format!("warm start: completion failed (share {share}): {reason}")); - } - if let Ok(completed) = completed { - debug(|| { - format!( - "warm start: plan (share {share}, amortized {amortized}) costs {:e}, {} shareable selected", - completed.cost_per_second, - plan.selected.len() - ) - }); - if best - .as_ref() - .is_none_or(|known| completed.cost_per_second < known.cost_per_second) - { - best = Some(StartingPlan { - values: completion.values(&completed), - cost_per_second: completed.cost_per_second, - }); - } - } - } - } - // Greedy sharing: prefer one more shareable candidate that several - // queries may read (most readers first), kept when the exact cost - // falls. - let mut sharable: Vec<(usize, PhysicalCandidateId)> = milp - .inputs - .context_of_shareable - .keys() - .map(|&m| (estimates.readers[m.0], m)) - .filter(|&(readers, _)| readers >= 2) - .collect(); - sharable.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); - let mut preferred: BTreeSet = BTreeSet::new(); - for (_, m) in sharable { - if best.is_none() { - break; - } - let mut trial = preferred.clone(); - trial.insert(m); - tried += 1; - let Some(plan) = sequential_plan(milp, &estimates, &orders[0], true, false, &trial) else { - continue; - }; - if let Ok(completed) = completion.complete(&binaries_of(milp, &plan)) { - if best - .as_ref() - .is_some_and(|known| completed.cost_per_second < known.cost_per_second) - { - preferred = trial; - best = Some(StartingPlan { - values: completion.values(&completed), - cost_per_second: completed.cost_per_second, - }); - } - } - } - debug(|| { - format!( - "warm start: {tried} plans tried, best {:?}", - best.as_ref().map(|best| best.cost_per_second) - ) - }); - best -} - -/// The pseudo-random orders tried after workload order and its reverse. -const MORE_ORDERS: usize = 30; - -/// The plan's binaries: `p[ctx, c]` and `x[m]` set to 1. -fn binaries_of(milp: &SelectionMilp<'_>, plan: &Plan) -> Vec<(Variable, f64)> { - let mut binaries = Vec::new(); - for (context, chosen) in plan.chosen.iter().enumerate() { - for candidate in chosen.values() { - if let Some(&variable) = milp.choice[context].get(candidate) { - binaries.push((variable, 1.0)); - } - } - } - for shareable in &plan.selected { - if let Some(&variable) = milp.selected.get(shareable) { - binaries.push((variable, 1.0)); - } - } - for (key, slot) in &plan.slot_reads { - if let Some(links) = milp.slot_links.get(key) { - for &(at, link) in links { - if at == *slot { - binaries.push((link, 1.0)); - } - } - } - } - binaries -} - -/// Every query planned in `order` into one plan; with `share`, what is -/// already selected costs nothing to read. -fn sequential_plan( - milp: &SelectionMilp<'_>, - estimates: &Estimates, - order: &[usize], - share: bool, - amortized: bool, - preferred: &BTreeSet, -) -> Option { - let mut plan = Plan { - chosen: vec![BTreeMap::new(); milp.choice.len()], - selected: BTreeSet::new(), - slot_reads: BTreeMap::new(), - }; - for &query in order { - let attempts = [ - (Objective::Cost, Approximate::Usable), - (Objective::Cost, Approximate::SizedFor(2)), - (Objective::Cost, Approximate::SizedFor(4)), - (Objective::Cost, Approximate::None), - (Objective::Latency, Approximate::Usable), - (Objective::Latency, Approximate::SizedFor(2)), - (Objective::Latency, Approximate::SizedFor(4)), - (Objective::Latency, Approximate::None), - ]; - // The first attempt meeting accuracy and the latency estimate; else - // the last meeting accuracy (the latency estimate is from each - // candidate's first costed usage, so it may be pessimistic: the - // completion checks the exact latency). - let mut planned = false; - let mut fallback = None; - for (attempt, (objective, approximate)) in attempts.into_iter().enumerate() { - let mut deriving = Derivation { - milp, - estimates, - query, - objective, - approximate, - share, - amortized, - preferred, - undo: Vec::new(), - steps: 0, - references: output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .map(|(reference, _)| reference) - .collect(), - }; - let derived = deriving.derive_query(&mut plan).is_ok(); - let accurate = derived && meets_accuracy(milp, &plan, query); - let fast = accurate && meets_latency(milp, &plan, query); - debug(|| { - format!( - "warm start: q{query} {objective:?}/{approximate:?}: derived {derived}, \ - accuracy {accurate}, latency {fast}, steps {}", - deriving.steps - ) - }); - if fast { - planned = true; - break; - } - if accurate { - fallback = Some(attempt); - } - let undo = std::mem::take(&mut deriving.undo); - undo_all(&mut plan, undo); - } - if let (false, Some(attempt)) = (planned, fallback) { - let (objective, approximate) = attempts[attempt]; - let mut deriving = Derivation { - milp, - estimates, - query, - objective, - approximate, - share, - amortized, - preferred, - undo: Vec::new(), - steps: 0, - references: output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .map(|(reference, _)| reference) - .collect(), - }; - planned = deriving.derive_query(&mut plan).is_ok(); - } - if !planned { - return None; - } - } - Some(plan) -} - -fn undo_all(plan: &mut Plan, undo: Vec) { - for change in undo.into_iter().rev() { - match change { - Undo::Chose(context, slot) => { - plan.chosen[context].remove(&slot); - } - Undo::Selected(shareable) => { - plan.selected.remove(&shareable); - } - Undo::SlotRead(context, reader, input) => { - plan.slot_reads.remove(&(context, reader, input)); - } - } - } -} - -/// Per context and candidate, the estimated cost and latency of the -/// cheapest subtree, per query (the rate of a query context; shared -/// contexts at the slowest rate of the queries that may reach them). -struct Estimates { - /// Per context, its rate (executions per second). - rate: Vec, - /// Per candidate, its numbers (first costed context). - numbers: Vec>, - /// Per candidate, the queries that may depend on it (shareable ones). - readers: Vec, -} - -impl Estimates { - fn new(milp: &SelectionMilp<'_>) -> Self { - let horizon = milp.cost_inputs.non_repeating_query_horizon_seconds; - let query_rate: Vec = milp - .entries - .iter() - .map(|entry| { - evaluation_rate([(&entry.recurrence, entry.range_evaluation)], horizon) - .unwrap_or(1.0) - }) - .collect(); - let rate = milp - .inputs - .contexts - .iter() - .map(|context| match context.kind { - ContextKind::Query(query) => query_rate[query], - ContextKind::Ingestion(_) => 1.0, - ContextKind::Shared(m) if context.at_query_time => { - let readers = milp.queries_reaching(m); - readers - .iter() - .map(|&query| query_rate[query]) - .fold(f64::INFINITY, f64::min) - .min(1e9) - } - ContextKind::Shared(_) => 1.0, - }) - .collect(); - let numbers = milp - .inputs - .candidate_data - .iter() - .map(|data| data.usage.as_ref().ok().map(NodeNumbers::of)) - .collect(); - let mut readers = vec![0; milp.inputs.candidate_data.len()]; - for &m in milp.inputs.context_of_shareable.keys() { - readers[m.0] = milp.queries_reaching(m).len(); - } - Self { - rate, - numbers, - readers, - } - } -} - -/// One query's derivation into a plan. -struct Derivation<'m, 'a> { - milp: &'m SelectionMilp<'a>, - estimates: &'m Estimates, - query: usize, - objective: Objective, - approximate: Approximate, - share: bool, - /// Divide a shareable candidate's own cost among the queries that may - /// reach it. - amortized: bool, - /// Shareable candidates estimated free to read (greedy sharing). - preferred: &'m BTreeSet, - undo: Vec, - steps: usize, - /// The query's output references. - references: Vec, -} - -impl Derivation<'_, '_> { - /// Whether the query may use `candidate`. - fn allowed(&self, candidate: PhysicalCandidateId) -> bool { - let inputs = self.milp.inputs; - if inputs.candidate_data[candidate.0].approximate { - if self.approximate == Approximate::None - || self.milp.entries[self.query] - .requirements - .accuracy - .is_exact() - { - return false; - } - if let Some(AccuracyWeight::Unusable(_)) = - inputs.accuracy_weights[self.query].get(&candidate) - { - return false; - } - if let Approximate::SizedFor(shares) = self.approximate { - if inputs.candidate_data[candidate.0].accuracy_budget_parts < shares { - return false; - } - } - } - // A candidate needing two or more reading queries is not planned - // alone; one some of the query's outputs may not reach is not used. - let references = &self.references; - !self - .milp - .physical - .reader_constraints - .iter() - .any(|constraint| match constraint { - ReaderConstraint::MinimumQueryReaders { - candidate: c, - minimum, - .. - } => *c == candidate && *minimum > 1, - ReaderConstraint::AllowedOutputs { - candidate: c, - outputs, - } => *c == candidate && references.iter().any(|r| !outputs.contains(r)), - ReaderConstraint::IncompatibleOutputs { - candidate: c, - left, - right, - } => *c == candidate && references.contains(left) && references.contains(right), - ReaderConstraint::HighestSharedQueryMaterialization { .. } => false, - }) - } - - /// The estimated cost (or latency) of `candidate`'s own run read in - /// `context`, plus the cheapest subtrees of its inputs. - fn estimate( - &self, - plan: &Plan, - memo: &mut HashMap<(usize, PhysicalCandidateId), f64>, - context: usize, - candidate: PhysicalCandidateId, - ) -> f64 { - if let Some(&known) = memo.get(&(context, candidate)) { - return known; - } - memo.insert((context, candidate), f64::INFINITY); - let milp = self.milp; - let inputs = milp.inputs; - let value = (|| { - if !self.allowed(candidate) { - return f64::INFINITY; - } - let Some(numbers) = self.estimates.numbers[candidate.0] else { - return f64::INFINITY; - }; - let kind = inputs.kind[candidate.0]; - // Where its own run is and where it reads its inputs. - let (run_context, already) = if milp.computes(context, candidate) { - (context, false) - } else { - match kind { - RunKind::Shareable => match milp.context_of(candidate) { - Some(own) => (own, plan.selected.contains(&candidate)), - None => return f64::INFINITY, - }, - RunKind::Ingestion => { - let Some(ingestion) = milp - .ingestion_context_read_by(context) - .filter(|&at| milp.choice[at].contains_key(&candidate)) - else { - return f64::INFINITY; - }; - let chosen = plan.chosen[ingestion].get(&inputs.slot[candidate.0]); - (ingestion, chosen == Some(&candidate)) - } - RunKind::PerContext => return f64::INFINITY, - } - }; - if (self.share && already) || self.preferred.contains(&candidate) { - return 0.0; - } - let mut own = match self.objective { - Objective::Cost => { - numbers.execution_cost * self.estimates.rate[run_context] - + numbers.standing_cost - } - Objective::Latency => numbers.latency_ms, - }; - if self.amortized && kind == RunKind::Shareable && self.objective == Objective::Cost { - own /= self.estimates.readers[candidate.0].max(1) as f64; - } - let mut total = own; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - let best = producers - .iter() - .filter(|producer| milp.choice[run_context].contains_key(producer)) - .filter(|producer| { - kind == RunKind::Shareable - || inputs.slot[producer.0] != inputs.slot[candidate.0] - }) - .map(|&producer| self.estimate(plan, memo, run_context, producer)) - .fold(f64::INFINITY, f64::min); - total += best; - } - total - })(); - memo.insert((context, candidate), value); - value - } - - fn derive_query(&mut self, plan: &mut Plan) -> Result<(), ()> { - let milp = self.milp; - let mut memo = HashMap::new(); - for (reference, (query, _)) in output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - { - if query != self.query { - continue; - } - let allowed: Vec = milp.inputs.output_candidates[reference] - .iter() - .copied() - .filter(|candidate| milp.choice[query].contains_key(candidate)) - .collect(); - self.read(plan, &mut memo, query, &allowed)?; - } - Ok(()) - } - - /// Makes `context` give one of `allowed` (one input's producers, or an - /// output's candidates). - fn read( - &mut self, - plan: &mut Plan, - memo: &mut HashMap<(usize, PhysicalCandidateId), f64>, - context: usize, - allowed: &[PhysicalCandidateId], - ) -> Result { - self.steps += 1; - if self.steps > MOST_STEPS_PER_QUERY { - return Err(()); - } - let milp = self.milp; - let inputs = milp.inputs; - // A slot already chosen here with an allowed candidate. - for &candidate in allowed { - if plan.chosen[context].get(&inputs.slot[candidate.0]) == Some(&candidate) { - return Ok(candidate); - } - } - let mut options: Vec<(f64, PhysicalCandidateId)> = allowed - .iter() - .copied() - .filter(|candidate| milp.choice[context].contains_key(candidate)) - .filter(|candidate| !plan.chosen[context].contains_key(&inputs.slot[candidate.0])) - .map(|candidate| (self.estimate(plan, memo, context, candidate), candidate)) - .collect(); - options.retain(|(estimate, _)| estimate.is_finite()); - options.sort_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1))); - for (_, candidate) in options { - let mark = self.undo.len(); - if self.choose(plan, memo, context, candidate).is_ok() { - return Ok(candidate); - } - let undo = self.undo.split_off(mark); - undo_all(plan, undo); - } - Err(()) - } - - /// Chooses `candidate` for its slot in `context` and derives what it - /// needs. - fn choose( - &mut self, - plan: &mut Plan, - memo: &mut HashMap<(usize, PhysicalCandidateId), f64>, - context: usize, - candidate: PhysicalCandidateId, - ) -> Result<(), ()> { - let milp = self.milp; - let inputs = milp.inputs; - let slot = inputs.slot[candidate.0]; - plan.chosen[context].insert(slot, candidate); - self.undo.push(Undo::Chose(context, slot)); - if milp.computes(context, candidate) { - for (input, producers) in inputs.producer_candidates_per_input[candidate.0] - .iter() - .enumerate() - { - let producers: Vec = producers - .iter() - .copied() - .filter(|producer| inputs.slot[producer.0] != slot) - .collect(); - let read = self.read(plan, memo, context, &producers)?; - if milp.slot_links.contains_key(&(context, candidate, input)) { - plan.slot_reads - .insert((context, candidate, input), inputs.slot[read.0]); - self.undo.push(Undo::SlotRead(context, candidate, input)); - } - } - return Ok(()); - } - match inputs.kind[candidate.0] { - RunKind::Shareable => { - if plan.selected.contains(&candidate) { - return Ok(()); - } - plan.selected.insert(candidate); - self.undo.push(Undo::Selected(candidate)); - let own = milp.context_of(candidate).ok_or(())?; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - self.read(plan, memo, own, producers)?; - } - Ok(()) - } - RunKind::Ingestion => { - let ingestion = milp.ingestion_context_read_by(context).ok_or(())?; - if context == ingestion { - return Err(()); - } - self.read(plan, memo, ingestion, &[candidate]).map(|_| ()) - } - RunKind::PerContext => Err(()), - } - } -} - -/// The runs query `query` depends on in `plan`: `(context, candidate)` for -/// candidates computed in its context and in the contexts of the -/// shareable candidates it reaches, and each such shareable candidate -/// `(its context, itself)`. -fn runs_reached( - milp: &SelectionMilp<'_>, - plan: &Plan, - query: usize, -) -> BTreeSet<(usize, PhysicalCandidateId)> { - let inputs = milp.inputs; - let mut reached = BTreeSet::new(); - let mut stack: Vec<(usize, PhysicalCandidateId)> = Vec::new(); - for (reference, (q, _)) in output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - { - if q != query { - continue; - } - for &candidate in &inputs.output_candidates[reference] { - if plan.chosen[query].get(&inputs.slot[candidate.0]) == Some(&candidate) { - stack.push((query, candidate)); - } - } - } - while let Some((context, candidate)) = stack.pop() { - let (run_context, own) = if milp.computes(context, candidate) { - (context, false) - } else { - match inputs.kind[candidate.0] { - RunKind::Shareable => match milp.context_of(candidate) { - Some(own) => (own, true), - None => continue, - }, - RunKind::Ingestion => match milp.ingestion_context_read_by(context) { - Some(ingestion) => (ingestion, false), - None => continue, - }, - RunKind::PerContext => continue, - } - }; - if !reached.insert((run_context, candidate)) { - continue; - } - let slot = inputs.slot[candidate.0]; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - for &producer in producers { - if !own && inputs.slot[producer.0] == slot { - continue; - } - if plan.chosen[run_context].get(&inputs.slot[producer.0]) == Some(&producer) { - stack.push((run_context, producer)); - break; - } - } - } - } - reached -} - -/// Whether query `query`'s approximate runs in `plan` meet the A1 rows. -fn meets_accuracy(milp: &SelectionMilp<'_>, plan: &Plan, query: usize) -> bool { - let inputs = milp.inputs; - let runs: Vec = runs_reached(milp, plan, query) - .into_iter() - .map(|(_, candidate)| candidate) - .filter(|candidate| inputs.candidate_data[candidate.0].approximate) - .filter(|candidate| inputs.accuracy_weights[query].contains_key(candidate)) - .collect(); - if runs.is_empty() { - return true; - } - if milp.entries[query].requirements.accuracy.is_exact() { - return false; - } - let count = runs.len(); - for &candidate in &runs { - if matches!( - inputs.accuracy_weights[query].get(&candidate), - Some(AccuracyWeight::Unusable(_)) - ) { - return false; - } - if inputs.candidate_data[candidate.0].accuracy_budget_parts < count { - return false; - } - } - !inputs.accuracy_excluded_pairs[query] - .iter() - .any(|(first, second)| runs.contains(first) && runs.contains(second)) -} - -/// Whether query `query`'s estimated latency in `plan` is within its bound. -fn meets_latency(milp: &SelectionMilp<'_>, plan: &Plan, query: usize) -> bool { - let Some(bound) = latency_bound_ms(&milp.entries[query]) else { - return true; - }; - let request_ms = milp.cost_inputs.measured_operation_costs.ns_per_evaluation * 1e-6; - let latency: f64 = runs_reached(milp, plan, query) - .into_iter() - .filter(|&(context, _)| milp.runs_at_query_time(context)) - .filter_map(|(_, candidate)| milp.inputs.candidate_data[candidate.0].usage.as_ref().ok()) - .map(|usage| usage.own_latency_ms) - .sum(); - debug(|| { - format!( - "warm start: q{query} estimated latency {} ms, bound {bound} ms", - latency + request_ms - ) - }); - latency + request_ms <= bound -} - -/// Prints `message` to stderr when `ASAP_STAGE3_DEBUG` is set. -pub(crate) fn debug(message: impl FnOnce() -> String) { - if std::env::var_os("ASAP_STAGE3_DEBUG").is_some() { - eprintln!("{}", message()); - } -} diff --git a/crates/plan-selection/src/models.rs b/crates/plan-selection/src/models.rs deleted file mode 100644 index bbc0d88f0..000000000 --- a/crates/plan-selection/src/models.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! The models Stage 3 checks and compares with: [`AccuracyCheckInputs`] -//! (validity) and [`CostInputs`] (the objective), and the engine's view of -//! both, [`SelectionModels`]. - -use asap_accuracy::{AccuracyEvidenceProvider, AccuracyModel}; -use asap_cost::CostInputs; - -/// Inputs of the composed-accuracy check. `raw_data_retained` is a Stage 2 -/// input only; to ignore accuracy guarantees the facade passes an -/// accept-all `model` ([`asap_accuracy::IgnoreAccuracyGuarantees`]). -#[derive(Clone, Copy)] -pub struct AccuracyCheckInputs<'a> { - /// Decides whether a candidate's accuracy guarantee meets its query's - /// target, and whether the deployment accepts an output regardless of - /// its guarantee. - pub model: &'a dyn AccuracyModel, - /// Deployment facts the guarantees need (operand domains, observed - /// guarantees). - pub evidence: &'a dyn AccuracyEvidenceProvider, -} - -/// Stage 3's accuracy model and evidence with its cost inputs, as the -/// engine reads them. -#[derive(Clone, Copy)] -pub(crate) struct SelectionModels<'a> { - pub accuracy: &'a dyn AccuracyModel, - pub evidence: &'a dyn AccuracyEvidenceProvider, - pub cost_inputs: &'a CostInputs, -} - -impl<'a> SelectionModels<'a> { - pub(crate) fn new( - accuracy_checks: &AccuracyCheckInputs<'a>, - cost_inputs: &'a CostInputs, - ) -> Self { - Self { - accuracy: accuracy_checks.model, - evidence: accuracy_checks.evidence, - cost_inputs, - } - } - - /// The built-in accuracy model and no evidence, under default cost - /// inputs. - #[cfg(test)] - pub(crate) fn builtin() -> SelectionModels<'static> { - static DEFAULT_COST_INPUTS: std::sync::LazyLock = - std::sync::LazyLock::new(CostInputs::default); - SelectionModels { - accuracy: &asap_accuracy::BuiltinAccuracyModel, - evidence: &asap_accuracy::NoAccuracyEvidence, - cost_inputs: &DEFAULT_COST_INPUTS, - } - } - - #[cfg(test)] - pub(crate) fn with_accuracy(mut self, accuracy: &'a dyn AccuracyModel) -> Self { - self.accuracy = accuracy; - self - } - - #[cfg(test)] - pub(crate) fn with_evidence(mut self, evidence: &'a dyn AccuracyEvidenceProvider) -> Self { - self.evidence = evidence; - self - } - - #[cfg(test)] - pub(crate) fn with_cost_inputs(mut self, cost_inputs: &'a CostInputs) -> Self { - self.cost_inputs = cost_inputs; - self - } -} diff --git a/crates/plan-selection/src/report.rs b/crates/plan-selection/src/report.rs index e2617b580..e20bb2a60 100644 --- a/crates/plan-selection/src/report.rs +++ b/crates/plan-selection/src/report.rs @@ -1,17 +1,12 @@ -//! What plan selection reports: the selection, each rejected candidate and why, -//! and the search report. - -use std::collections::BTreeMap; - -use asap_cost::analytical::AnalyticalCostError; -use asap_cost::{CostModel, PlanCost}; -use asap_types::ir::physical_export::PhysicalASAPNodeId; -use asap_types::ir::properties::AccuracyMetric; -#[cfg(any(test, feature = "reference-pricer"))] -use asap_types::ir::QueryRoot; -use asap_types::stages::{ - DagOfSelectedCandidates, PhysicalCandidateId, PrunedCandidate, SelectedNodeId, -}; +//! What plan selection reports beside the plan: the accepted accuracy, the +//! candidates it pruned and the selection MILP's size and time; and each +//! rejected candidate and why. + +use asap_cost::node::analytical::AnalyticalCostError; +use asap_cost::CostModel; +use asap_types::accuracy::AccuracyMetric; +use asap_types::plan::physical_asap_dag::PhysicalASAPNodeId; +use asap_types::stages::{PhysicalCandidate, PrunedCandidate}; use thiserror::Error; /// Peak memory above which a candidate's report carries a warning when the @@ -47,7 +42,7 @@ pub struct Rejection { pub reason: String, } -/// Why Stage 3 did not select a candidate. The first six mean the +/// Why Stage 3 did not select a candidate. The first five mean the /// candidate failed a check and is invalid ([`Self::failed_a_check`]); the /// rest mean it was valid and lost under the cost model. JSON: the /// snake_case variant name, e.g. `"costlier"`. @@ -57,15 +52,8 @@ pub enum RejectionCause { /// A query's accuracy target is not met: its output's guarantee misses /// it and no deployment evidence meets it. AccuracyTargetMissed, - /// A query's plan reads a Count-Min sketch whose update weights are not - /// proven non-negative (e.g. samples of a metric declared neither a - /// counter nor non-negative). The executor cannot run it, so the deployment's accuracy - /// model cannot accept it either. - CountMinWeightsNotProvenNonNegative, - /// A query's output has no accuracy guarantee (Stage 1 has no - /// analytical rule for it and the deployment's evidence supplies none), - /// and the deployment's accuracy model did not accept it directly - /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`](asap_accuracy::AccuracyModel::accepts_output_regardless_of_guarantee)). + /// A query's output has no accuracy guarantee: no analytical rule and + /// no deployment evidence supplies one. NoAccuracyGuarantee, /// A query's latency is over its `max_latency_ms` bound. LatencyBoundExceeded, @@ -76,9 +64,9 @@ pub enum RejectionCause { /// Under cost model 3, its usage is over one of the cluster's limits. OverClusterCapacity, /// Its cost overflows the cost model's arithmetic, so it cannot be - /// priced. + /// costed. CostNotRepresentable, - /// Under cost models 1 and 2, its price is higher than the selected + /// Under cost models 1 and 2, its cost is higher than the selected /// candidate's. Costlier, /// Under cost model 3's `lowest-latency`, it is slower than the @@ -91,7 +79,7 @@ pub enum RejectionCause { /// order. TiedLostOnEnumerationOrder, /// No plan meets every query's accuracy target, latency bound and the - /// catalog's reader constraints together: the selection MILP is + /// catalog's consumer constraints together: the selection MILP is /// infeasible. The reason names a query when it has no plan even alone. Infeasible, } @@ -104,7 +92,6 @@ impl RejectionCause { matches!( self, Self::AccuracyTargetMissed - | Self::CountMinWeightsNotProvenNonNegative | Self::NoAccuracyGuarantee | Self::LatencyBoundExceeded | Self::LatencyOverRepeatInterval @@ -115,66 +102,30 @@ impl RejectionCause { } } -/// The accuracy Stage 3 accepted for one query of a valid candidate: the -/// metric the query's target set (if any) and the metric of the guarantee -/// that met it. With no metric set, any metric meeting the target's ε, δ -/// and scope is accepted, so ε means different things across candidates -/// (e.g. 0.01 of the true value vs 0.01 of the rank); this record makes -/// the accepted metric visible. +/// The accuracy Stage 3 accepted for one query: the metric of the error +/// bound that met its target, and the sketches its output reads. #[derive(Debug, Clone, PartialEq)] pub struct AcceptedAccuracy { - /// The target's metric; `None` when the user did not set one. - pub target_metric: Option, - /// The metric the accepted guarantee is stated in; `None` for an exact - /// output, which meets every metric, and for one the deployment's model - /// accepted regardless of its guarantee (no guarantee was checked). + /// The metric of the accepted error bound; `None` for an exact output, + /// which meets every metric, and for a query without an accuracy target. pub accepted_metric: Option, - /// Whether the deployment's accuracy model accepted the output directly, - /// whatever its guarantee says - /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`](asap_accuracy::AccuracyModel::accepts_output_regardless_of_guarantee)), as - /// every output is with [`IgnoreAccuracyGuarantees`](asap_accuracy::IgnoreAccuracyGuarantees); `false` when its - /// guarantee was checked and met the target. - pub accepted_regardless_of_guarantee: bool, - /// When accepted regardless of its guarantee, the cause the guarantee - /// check would have rejected it with - /// ([`RejectionCause::NoAccuracyGuarantee`] or - /// [`RejectionCause::AccuracyTargetMissed`]): the output was accepted - /// only because of the accuracy model (e.g. - /// [`IgnoreAccuracyGuarantees`](asap_accuracy::IgnoreAccuracyGuarantees)). `None` when - /// its guarantee meets the target. - pub guarantee_check_rejection: Option, /// The sketch algorithms the query's output reads, in plan order /// (e.g. `["Kll"]`); empty for an exact plan. pub sketches: Vec, } impl std::fmt::Display for AcceptedAccuracy { - /// E.g. `metric not set; accepted rank (Kll)`, or - /// `metric relative; accepted relative (DDSketch)`, or - /// `metric rank; accepted by the deployment's accuracy model regardless of guarantee (Kll)`. + /// E.g. `accepted rank (Kll)`, or `accepted exact`. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let name = |metric: AccuracyMetric| { - serde_json::to_value(metric) - .ok() - .and_then(|value| value.as_str().map(str::to_owned)) - .unwrap_or_else(|| format!("{metric:?}")) - }; - match self.target_metric { - None => write!(f, "metric not set; ")?, - Some(metric) => write!(f, "metric {}; ", name(metric))?, - } match self.accepted_metric { - _ if self.accepted_regardless_of_guarantee => { - write!( - f, - "accepted by the deployment's accuracy model regardless of guarantee" - )?; - if let Some(cause) = self.guarantee_check_rejection { - write!(f, ", ignoring a failed guarantee check ({cause:?})")?; - } - } None => write!(f, "accepted exact")?, - Some(metric) => write!(f, "accepted {}", name(metric))?, + Some(metric) => { + let name = serde_json::to_value(metric) + .ok() + .and_then(|value| value.as_str().map(str::to_owned)) + .unwrap_or_else(|| format!("{metric:?}")); + write!(f, "accepted {name}")? + } } if !self.sketches.is_empty() { write!(f, " ({})", self.sketches.join(", "))?; @@ -183,109 +134,22 @@ impl std::fmt::Display for AcceptedAccuracy { } } -/// The whole-plan selector's result (`PlanSelector`): -/// the selected candidate among whole physical plans and the plan to run, -/// with what it kept of the other candidates. The reference pricer (feature -/// `reference-pricer`) selects among enumerated plans with it. Keeping -/// every candidate, `costs` and `accepted` list every valid candidate and -/// `rejected` every other one; otherwise only the selected candidate is -/// kept, and `rejected` holds the first rejection of each cause. -#[cfg(any(test, feature = "reference-pricer"))] -#[derive(Debug, Clone, PartialEq)] -pub struct WholePlanSelection { - pub selected: String, - /// The selected candidate's query roots with every weight requirement - /// Stage 3 confirmed made a proof: the plan to run. The Stage 2 - /// candidate keeps the open requirement. - pub plan: Vec, - pub costs: BTreeMap, - /// For each valid candidate, per query, the accuracy Stage 3 accepted. - pub accepted: BTreeMap>, - pub rejected: Vec, - /// The selected plan's tumbling-window retention, by node id in its physical DAG - /// ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)). - pub tumbling_window_retention_ms: BTreeMap, - /// The physical candidates checked, valid or not. - pub candidates_checked: u64, -} - -/// Stage 3's result ([`select_plan`](crate::select_plan)): the selected DAG -/// over the [`PhysicalASAPMemoCandidates`](asap_types::stages::PhysicalASAPMemoCandidates) -/// it was given, its cost and accepted accuracy, and the search report. -/// The facade builds the plan from `selected_dag` with the same -/// [`AssignmentRealizer`](asap_types::stages::AssignmentRealizer). +/// What Stage 3 reports about its own run, beside the plan +/// ([`select_plan`](crate::select_plan)). The plan's cost is +/// `asap_cost::plan_cost` of the plan; this does not repeat it. #[derive(Debug, Clone, PartialEq)] pub struct Selection { - /// The selected plan: one node per run of a candidate. - pub selected_dag: DagOfSelectedCandidates, - /// Nodes of `selected_dag` whose open count-min weight requirement - /// Stage 3 proved; the facade applies the proof to each node's operator - /// node when it builds the plan. - pub nodes_with_proven_non_negative_weights: Vec, - pub cost: PlanCost, /// Per query, the accuracy Stage 3 accepted for the selected plan. pub accepted_accuracy_per_query: Vec, - /// The candidates Stage 3 ruled out before the search: dominated ones and - /// those whose usage could not be costed. - pub pruned_candidates: Vec>, - pub search_report: SearchReport, -} - -/// How the search behind a selection ended, and what it cost. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct SearchReport { - pub status: SearchStatus, - /// The solver's lower bound on the selected plan's cost per second; - /// `None` when the solver's dual bound is not finite. - pub cost_lower_bound_per_second: Option, - /// The largest MILP the search built. + /// The candidates Stage 3 ruled out before the selection MILP: + /// dominated ones and those whose usage could not be costed. + pub pruned_candidates: Vec>, + /// The selection MILP's size. pub milp_variables: usize, pub milp_constraints: usize, - pub model_construction_ms: u64, - pub solver_ms: u64, - pub validation_ms: u64, - /// Set when `asap_cost::plan_cost` of the selected plan differs from - /// the selection MILP's objective and the model encoded some cost terms - /// inexactly (open questions on #1095, e.g. the working-memory term), so - /// the plan is returned with its cost equality unproven. `None` when - /// the two are equal (within a relative 10⁻⁶). A - /// difference with no inexact term is an error - /// ([`ErrorSelectingPlan::ObjectiveDiffersFromPlanCost`]). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub inexact_objective: Option, -} - -/// The selected plan's cost per second as the selection MILP's objective -/// and as `asap_cost::plan_cost` computes it (both without request fees), -/// and the cost terms the model encoded inexactly, with how many times. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct InexactObjective { - pub objective_per_second: f64, - pub plan_cost_per_second: f64, - pub inexact_terms: BTreeMap, -} - -impl SearchReport { - /// How far the lower bound is below `cost_per_second`, as a fraction - /// of it in [0, 1]; `None` without a bound. - pub fn relative_gap(&self, cost_per_second: f64) -> Option { - self.cost_lower_bound_per_second.map(|bound| { - if cost_per_second == 0.0 { - 0.0 - } else { - ((cost_per_second - bound).max(0.0) / cost_per_second.abs()).min(1.0) - } - }) - } -} - -/// How the search ended with a validated plan. JSON: snake_case. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SearchStatus { - /// The search proved the plan optimal over the catalog it was given - /// (the only outcome: the solve has no time limit). - Optimal, + /// How long building and solving the selection MILP took. + pub milp_construction_ms: u64, + pub milp_solve_ms: u64, } /// Why Stage 3 selected no plan. @@ -311,36 +175,12 @@ pub enum ErrorSelectingPlan { }, #[error("no valid candidate: {0:?}")] NoValidCandidate(Vec), - /// Stage 2 rejected a plan Stage 3 decoded from its catalog: a Stage 2 - /// catalog bug. - #[error("the realizer rejected the selected plan: {0}")] - RealizerRejected(String), /// The memo search's MILP solver failed. #[error("MILP solver: {0}")] Solver(String), /// The selection MILP implements cost model 1 only (#1095 §3). #[error("the selection MILP does not support {0}")] UnsupportedCostModel(String), - /// `asap_cost::plan_cost` of the selected plan differs from the - /// selection MILP's objective (both without request fees) and the model - /// encoded every cost term exactly: a bug in a cost term. - #[error( - "the selection MILP's objective {objective_per_second:e} differs from plan_cost \ - {plan_cost_per_second:e} (per second, without request fees)" - )] - ObjectiveDiffersFromPlanCost { - objective_per_second: f64, - plan_cost_per_second: f64, - }, - /// The selected plan fails a check the selection MILP encodes - /// (accuracy through the budget shares, latency): a bug in the - /// accuracy weights, the sizing rule or a latency term, never a reason - /// to re-solve. - #[error("the selected plan fails a check the MILP encodes ({cause:?}): {reason}")] - SelectedPlanFailedCheck { - cause: RejectionCause, - reason: String, - }, } #[cfg(test)] diff --git a/crates/plan-selection/src/select.rs b/crates/plan-selection/src/select.rs index ed5546432..8eff2f98e 100644 --- a/crates/plan-selection/src/select.rs +++ b/crates/plan-selection/src/select.rs @@ -1,373 +1,33 @@ -//! Stage 3, plan selection: [`select_plan`] runs the group selection MILP -//! (#1095). Inputs it does not price yet (cost models 2 and 3, or a measured -//! operation cost table without a frozen calibration factor) are an -//! `UnsupportedCostModel` error (#888). -//! -//! One MILP solve and one realization, no loop: -//! 1. the MILP's inputs (candidate dominance, execution contexts, accuracy -//! weights) and the model, solved once with HiGHS from a warm start to -//! proven optimality (no time limit); -//! 2. the solution decoded into a [`DagOfSelectedCandidates`] -//! ([`crate::milp::decode`]) and validated against the catalog; -//! 3. the plan realized once through the injected realizer: a rejection is -//! a Stage 2 catalog bug ([`ErrorSelectingPlan::RealizerRejected`]); -//! 4. exact checks: `asap_cost::plan_cost` of the realized plan minus the -//! request fees equals the objective within a relative 10⁻⁶ (otherwise -//! an error, or, when the model encoded some term inexactly, -//! [`SearchReport::inexact_objective`]); every query's accuracy -//! (`satisfies`) and latency hold, or it is an error. -//! -//! With `ASAP_STAGE3_REPORT=`, one JSON line per run is appended -//! there: the model's size per family and per scope (the shareable -//! candidates' contexts apart), the timings, the warm start's cost, the -//! objective, `plan_cost` and the dual bound (measurement only). - -use std::collections::BTreeMap; -use std::time::Instant; - -use asap_cost::{kept_tumbling_window_chains, node_map, plan_cost, CostInputs}; -use asap_types::ir::{ASAPOp, Operator}; -use asap_types::stages::{ - AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, - PhysicalCandidateError, PhysicalCandidateId, PruneReason, PrunedCandidate, RealizedPlan, - SelectedNodeId, -}; -use asap_types::workload::bounds::confirmed_weights; +//! Stage 3, plan selection: [`select_plan`] builds the selection MILP over +//! Stage 2's physical candidates and admissible edges +//! ([`crate::milp::model`]), solves it, and builds the plan from the +//! selected executions as its last step. + +use asap_cost::CostInputs; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; +use asap_types::stages::PhysicalASAPMemoCandidates; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; -use crate::checks::{accuracy_check, accuracy_targets, latency_violation}; -use crate::groups::{prepare_milp_inputs_with, MilpInputs}; -use crate::milp::decode::{decode, ChosenCandidates}; -use crate::milp::model::{ - build_selection_milp, solve_selection_milp, SelectionMilpError, SelectionMilpOptions, -}; -use crate::models::{AccuracyCheckInputs, SelectionModels}; -use crate::report::{ - ErrorSelectingPlan, InexactObjective, Rejection, RejectionCause, SearchReport, SearchStatus, - Selection, -}; - -/// The largest relative difference between the objective and `plan_cost` -/// that counts as equal. -const COST_EQUALITY_TOLERANCE: f64 = 1e-6; +use crate::report::{ErrorSelectingPlan, Selection}; /// Stage 3, plan selection: select one plan among `physical`'s candidates -/// with the group selection MILP (module documentation). The selection is -/// realized through `realizer`, checked against every query's accuracy -/// target (`accuracy_checks`) and latency bound, and priced under -/// `cost_inputs`. `entries[q]` is query `q`'s workload entry (accuracy -/// target, latency bound, recurrence); `data` the data workload with its -/// bounds. The solve has no time limit: it runs to proven optimality. +/// (module documentation) under `cost_inputs`, and build it: one +/// `PhysicalASAPDAG` node per selected execution, its inputs from the +/// selected edges, each node's `selected_physical_candidate` set. +/// `entries[q]` is query `q`'s workload entry; `data` the data workload +/// with its bounds. /// -/// Err: invalid or unsupported cost inputs, an infeasible catalog, a -/// solver failure, or the realizer rejected the selected plan -/// ([`ErrorSelectingPlan::RealizerRejected`]: a Stage 2 catalog bug). +/// Err: invalid or unsupported cost inputs, an infeasible catalog, or a +/// solver failure. pub fn select_plan( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], data: &DataWorkload, - accuracy_checks: &AccuracyCheckInputs<'_>, cost_inputs: &CostInputs, - realizer: &dyn AssignmentRealizer, -) -> Result { +) -> Result<(PhysicalASAPDAG, Selection), ErrorSelectingPlan> { cost_inputs .validate() .map_err(ErrorSelectingPlan::InvalidSettings)?; - // The MILP's coefficients and `plan_cost` must use one calibration: a - // measured cost table needs a frozen factor. - if cost_inputs.frozen_calibration_factor.is_none() - && !cost_inputs.measured_operation_costs.operators.is_empty() - { - return Err(ErrorSelectingPlan::UnsupportedCostModel( - "a measured operation cost table without a frozen calibration factor".into(), - )); - } - let formula_scale = cost_inputs.frozen_calibration_factor.unwrap_or(1.0); - let models = SelectionModels::new(accuracy_checks, cost_inputs); - // The A1 rows decide accuracy only when the built-in rule does: a model - // or evidence that may accept an output otherwise is asked after the - // solve. - let enforce_accuracy = !(accuracy_checks.model.may_accept_regardless_of_guarantee() - || accuracy_checks.evidence.may_provide_evidence()); - - let construction_started = Instant::now(); - let inputs = prepare_milp_inputs_with(physical, entries, data, &models); - let pruned_candidates = pruned_before_the_milp(physical, &inputs); - let milp = build_selection_milp( - physical, - &inputs, - entries, - cost_inputs, - SelectionMilpOptions { - enforce_accuracy, - ..SelectionMilpOptions::default() - }, - ) - .map_err(selection_error)?; - let build_ms = construction_started.elapsed().as_millis() as u64; - let solution = solve_selection_milp(milp, build_ms).map_err(|error| { - with_uncostable_reasons(selection_error(error), &inputs, &pruned_candidates) - })?; - - let validation_started = Instant::now(); - let chosen = ChosenCandidates { - chosen: solution.chosen.clone(), - selected: solution.selected.clone(), - slot_reads: solution.slot_reads.clone(), - }; - let selected_dag = decode(physical, &inputs, &chosen).map_err(|reason| { - ErrorSelectingPlan::Solver(format!("the selection does not decode: {reason}")) - })?; - selected_dag.validate(physical).map_err(|reason| { - ErrorSelectingPlan::Solver(format!("the decoded selection is invalid: {reason}")) - })?; - let realized = realize(physical, &selected_dag, realizer)?; - let plan = &realized.plan; - let recurrence: Vec<_> = entries.iter().map(|e| e.recurrence.clone()).collect(); - let ranges: Vec<_> = entries.iter().map(|e| e.range_evaluation).collect(); - let (cost, _) = plan_cost( - &plan.dag, - &node_map(&plan.dag), - &kept_tumbling_window_chains(&plan.dag), - &recurrence, - &ranges, - data, - cost_inputs, - formula_scale, - ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: plan.id.clone(), - node, - error, - })?; - let plan_cost_per_second = cost.cost_per_second.ok_or_else(|| { - ErrorSelectingPlan::UnsupportedCostModel("a cost model without a cost per second".into()) - })? - solution.request_cost_per_second; - let objective_per_second = solution.cost_per_second; - let difference = (objective_per_second - plan_cost_per_second).abs(); - let tolerance = COST_EQUALITY_TOLERANCE * plan_cost_per_second.abs().max(f64::MIN_POSITIVE); - let within = difference <= tolerance; - report_run(&solution, objective_per_second, plan_cost_per_second); - let inexact_objective = if within { - None - } else if solution.size.inexact.is_empty() { - return Err(ErrorSelectingPlan::ObjectiveDiffersFromPlanCost { - objective_per_second, - plan_cost_per_second, - }); - } else { - Some(InexactObjective { - objective_per_second, - plan_cost_per_second, - inexact_terms: solution - .size - .inexact - .iter() - .map(|(reason, count)| (reason.to_string(), *count)) - .collect(), - }) - }; - let accepted_accuracy_per_query = - accuracy_check(plan, &accuracy_targets(entries), data, &models).map_err(|failure| { - ErrorSelectingPlan::SelectedPlanFailedCheck { - cause: failure.cause, - reason: failure.reason, - } - })?; - if let Some((cause, reason)) = latency_violation(&cost, entries) { - return Err(ErrorSelectingPlan::SelectedPlanFailedCheck { cause, reason }); - } - let nodes_with_proven_non_negative_weights = proven_non_negative_weights(&realized, data); - let validation_ms = validation_started.elapsed().as_millis() as u64; - Ok(Selection { - selected_dag, - nodes_with_proven_non_negative_weights, - cost, - accepted_accuracy_per_query, - pruned_candidates, - search_report: SearchReport { - status: SearchStatus::Optimal, - // The dual bound is on the objective, which leaves out the - // request fees every plan pays; the selection's cost has them. - cost_lower_bound_per_second: solution - .cost_lower_bound_per_second - .is_finite() - .then_some(solution.cost_lower_bound_per_second + solution.request_cost_per_second), - milp_variables: solution.size.variables, - milp_constraints: solution.size.rows, - model_construction_ms: solution.build_ms + solution.warm_start_ms, - solver_ms: solution.solve_ms, - validation_ms, - inexact_objective, - }, - }) -} - -/// The selection MILP's error as Stage 3's. -fn selection_error(error: SelectionMilpError) -> ErrorSelectingPlan { - let infeasible = |id: String, cause: RejectionCause, reason: String| { - ErrorSelectingPlan::NoValidCandidate(vec![Rejection { id, cause, reason }]) - }; - match error { - SelectionMilpError::UnsupportedCostModel(model) => { - ErrorSelectingPlan::UnsupportedCostModel(model.to_string()) - } - SelectionMilpError::EntryCount { roots, entries } => ErrorSelectingPlan::EntryCount { - candidate: "the physical memo".into(), - roots, - entries, - }, - SelectionMilpError::Infeasible => infeasible( - "workload".into(), - RejectionCause::Infeasible, - "no plan meets every query's accuracy target, latency bound and the reader \ - constraints together (the selection MILP is infeasible)" - .into(), - ), - SelectionMilpError::Cost(reason) | SelectionMilpError::Solver(reason) => { - ErrorSelectingPlan::Solver(reason) - } - } -} - -/// An infeasible selection with the reasons `asap-cost` could not cost -/// the candidates the model left out (e.g. a source missing the fact its -/// pricing needs), each once in candidate order: those candidates may be -/// why no plan is left. -fn with_uncostable_reasons( - error: ErrorSelectingPlan, - inputs: &MilpInputs, - pruned_candidates: &[PrunedCandidate], -) -> ErrorSelectingPlan { - let ErrorSelectingPlan::NoValidCandidate(mut rejections) = error else { - return error; - }; - let mut reasons: Vec<&str> = Vec::new(); - for pruned in pruned_candidates { - if pruned.reason != PruneReason::UsageNotCostable { - continue; - } - if let Err(reason) = &inputs.candidate_data[pruned.candidate.0].usage { - if !reasons.contains(&reason.as_str()) { - reasons.push(reason); - } - } - } - if let Some(rejection) = rejections.first_mut() { - if !reasons.is_empty() { - rejection.reason = format!( - "{}; candidates left out because they could not be costed: {}", - rejection.reason, - reasons.join("; ") - ); - } - } - ErrorSelectingPlan::NoValidCandidate(rejections) -} - -/// What Stage 3 removes before the MILP: dominated candidates, and those -/// `asap-cost` could not cost (the model leaves them out), in candidate -/// order. -fn pruned_before_the_milp( - physical: &PhysicalASAPMemoCandidates, - inputs: &MilpInputs, -) -> Vec> { - let mut reasons: BTreeMap = inputs - .pruned_candidates - .iter() - .map(|pruned| (pruned.class, PruneReason::DominatedBy(pruned.dominated_by))) - .collect(); - for (id, data) in inputs.candidate_data.iter().enumerate() { - if data.usage.is_err() { - reasons - .entry(PhysicalCandidateId(id)) - .or_insert(PruneReason::UsageNotCostable); - } - } - reasons - .into_iter() - .map(|(candidate, reason)| PrunedCandidate { - group: physical.memo_group_of(candidate), - candidate, - reason, - }) - .collect() -} - -/// `selected` realized through `realizer`; a rejection is a Stage 2 -/// catalog bug. -fn realize( - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, - realizer: &dyn AssignmentRealizer, -) -> Result { - realizer - .realize(physical, selected) - .map_err(|error| match error { - PhysicalCandidateError::RejectedAssignment(reason) => { - ErrorSelectingPlan::RealizerRejected(reason) - } - error => ErrorSelectingPlan::RealizerRejected(error.to_string()), - }) -} - -/// The selected nodes whose summary (`SummaryAgg` or `SummaryTumblingWindowChain`) reads -/// update weights `data` proves non-negative although the candidate leaves -/// the requirement open. -fn proven_non_negative_weights( - realized: &RealizedPlan, - data: &DataWorkload, -) -> Vec { - realized - .operator_node_of_selected_node - .iter() - .enumerate() - .filter(|(_, node)| match &node.operator { - Operator::ASAP( - ASAPOp::SummaryAgg { input, .. } | ASAPOp::SummaryTumblingWindowChain { input, .. }, - ) => confirmed_weights(&input.weight_domain, data) != input.weight_domain, - _ => false, - }) - .map(|(node, _)| SelectedNodeId(node)) - .collect() -} - -#[cfg(test)] -mod tests; - -/// Appends one JSON line describing this run to `ASAP_STAGE3_REPORT`, if -/// set (measurement only; failures to write are ignored). -fn report_run( - solution: &crate::milp::model::SelectionMilpSolution, - objective_per_second: f64, - plan_cost_per_second: f64, -) { - let Ok(path) = std::env::var("ASAP_STAGE3_REPORT") else { - return; - }; - let line = serde_json::json!({ - "variables": solution.size.variables, - "binaries": solution.size.binaries, - "rows": solution.size.rows, - "nonzeros": solution.size.nonzeros, - "scopes": solution.size.scopes.iter().map(|(scope, size)| (format!("{scope:?}"), size)).collect::>(), - "families": solution.size.families, - "inexact": solution.size.inexact, - "build_ms": solution.build_ms, - "warm_start_ms": solution.warm_start_ms, - "solve_ms": solution.solve_ms, - "warm_start_cost_per_second": solution.warm_start_cost_per_second, - "objective_per_second": objective_per_second, - "plan_cost_per_second": plan_cost_per_second, - "dual_bound_per_second": solution.cost_lower_bound_per_second, - "parts": solution.parts, - }); - use std::io::Write; - if let Ok(mut file) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(path) - { - let _ = writeln!(file, "{line}"); - } + let _ = (physical, entries, data); + todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution") } diff --git a/crates/plan-selection/src/select/tests.rs b/crates/plan-selection/src/select/tests.rs deleted file mode 100644 index 68ce48681..000000000 --- a/crates/plan-selection/src/select/tests.rs +++ /dev/null @@ -1,173 +0,0 @@ -use super::*; -use crate::milp::model::tests::{approximate, cost_inputs, shared_sum, through_stages, Fixture}; -use asap_accuracy::{AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence}; -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; -use asap_types::stages::RealizedPlan; - -fn realizer(fixture: &Fixture) -> impl AssignmentRealizer + '_ { - asap_physical_asap_optimization::Stage2AssignmentRealizer { - original_roots: &fixture.roots, - entries: &fixture.entries, - data: &fixture.data, - query_time_results_retained: fixture.query_time_results_retained, - } -} - -fn select( - fixture: &Fixture, - accuracy: &dyn AccuracyModel, - realizer: &dyn AssignmentRealizer, -) -> Result { - select_plan( - &fixture.physical, - &fixture.entries, - &fixture.data, - &AccuracyCheckInputs { - model: accuracy, - evidence: &NoAccuracyEvidence, - }, - &cost_inputs(), - realizer, - ) -} - -/// `fixture` with metric `m` declared non-negative. -fn non_negative(mut fixture: Fixture) -> Fixture { - let source = fixture.data.sources.get_mut("m").expect("m"); - source.values_non_negative = true; - fixture -} - -// The group selection MILP's selection: an optimal, valid plan whose cost is -// `plan_cost`'s, the MILP's size and every query's accepted accuracy. The -// ratio of quantiles is a regression of #1127: the A1 rows admitted two -// KLL sketches at parts = 2, whose rank errors compose no guarantee through -// the quotient, so the search returned `SelectedPlanFailedCheck`. -#[test] -fn the_milp_search_selects_a_checked_plan() { - let ratio = || { - through_stages( - &[approximate( - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - 60_000, - 0.01, - )], - true, - ) - }; - for fixture in [ - shared_sum(), - through_stages( - &[approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.01)], - true, - ), - ratio(), - non_negative(ratio()), - ] { - let stage2 = realizer(&fixture); - let selection = select(&fixture, &BuiltinAccuracyModel, &stage2).unwrap(); - selection.selected_dag.validate(&fixture.physical).unwrap(); - let report = &selection.search_report; - assert_eq!(report.status, SearchStatus::Optimal); - assert_eq!(report.inexact_objective, None); - assert!(report.milp_variables > 0 && report.milp_constraints > 0); - let cost = selection.cost.cost_per_second.unwrap(); - assert!(report.relative_gap(cost).unwrap() <= 1e-6, "{report:?}"); - assert_eq!( - selection.accepted_accuracy_per_query.len(), - fixture.entries.len() - ); - for pruned in &selection.pruned_candidates { - assert!(matches!( - pruned.reason, - PruneReason::DominatedBy(_) | PruneReason::UsageNotCostable - )); - assert_eq!( - pruned.group, - fixture.physical.memo_group_of(pruned.candidate) - ); - } - } -} - -/// A realizer that rejects every selection. -struct Rejecting; - -impl AssignmentRealizer for Rejecting { - fn realize( - &self, - _physical: &PhysicalASAPMemoCandidates, - _selected: &DagOfSelectedCandidates, - ) -> Result { - Err(PhysicalCandidateError::RejectedAssignment( - "a combination the catalog allows".into(), - )) - } -} - -// A realizer rejection is a Stage 2 catalog bug: an error, no re-solve. -#[test] -fn a_realizer_rejection_is_realizer_rejected() { - let fixture = shared_sum(); - match select(&fixture, &BuiltinAccuracyModel, &Rejecting) { - Err(ErrorSelectingPlan::RealizerRejected(reason)) => { - assert!(reason.contains("the catalog allows"), "{reason}") - } - other => panic!("{other:?}"), - } -} - -/// An accuracy model that accepts no guarantee, exact included, and never -/// accepts an output otherwise: the A1 rows still run, so the MILP's plan -/// misses `satisfies`, as it would with a wrong weight table. -struct AcceptsNothing; - -impl AccuracyModel for AcceptsNothing { - fn satisfies(&self, _guarantee: &AccuracyGuarantee, _target: &AccuracyTarget) -> bool { - false - } - - fn may_accept_regardless_of_guarantee(&self) -> bool { - false - } -} - -// A selected plan that misses `satisfies` is an error, never returned. -#[test] -fn an_accuracy_miss_is_an_error() { - let fixture = shared_sum(); - let stage2 = realizer(&fixture); - match select(&fixture, &AcceptsNothing, &stage2) { - Err(ErrorSelectingPlan::SelectedPlanFailedCheck { cause, .. }) => { - assert!(cause.failed_a_check(), "{cause:?}") - } - other => panic!("{other:?}"), - } -} - -// Cost models 2 and 3 are not supported by the group selection MILP (#888). -#[test] -fn cost_model_2_is_unsupported() { - let fixture = shared_sum(); - let costs = CostInputs { - cost_model: asap_cost::CostModel::ProvisionedCapacity( - asap_cost::ResourceUnitCost::illustrative(), - ), - ..cost_inputs() - }; - let result = select_plan( - &fixture.physical, - &fixture.entries, - &fixture.data, - &AccuracyCheckInputs { - model: &BuiltinAccuracyModel, - evidence: &NoAccuracyEvidence, - }, - &costs, - &Rejecting, - ); - assert!(matches!( - result, - Err(ErrorSelectingPlan::UnsupportedCostModel(_)) - )); -} diff --git a/crates/plan-selection/src/selector.rs b/crates/plan-selection/src/selector.rs deleted file mode 100644 index 5270385cd..000000000 --- a/crates/plan-selection/src/selector.rs +++ /dev/null @@ -1,471 +0,0 @@ -//! Choosing among whole checked plans: [`PlanSelector`] and -//! [`select_among_plans`]. The reference pricer (feature `reference-pricer`) -//! selects among enumerated plans with it. - -use std::collections::BTreeMap; - -use asap_cost::{ - worst_latency_ms, CalibrationFit, ClusterLimits, CostModel, PlanCost, PlanRanking, - ResourceUsage, COST_UNIT, -}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::workload::{DataWorkload, QueryRecurrence, QueryWorkloadEntry, RangeEvaluation}; - -use crate::checks::{accuracy_check, accuracy_targets, latency_violation}; -use crate::report::{ - AcceptedAccuracy, ErrorSelectingPlan, Rejection, RejectionCause, WholePlanSelection, -}; -use crate::SelectionModels; -use asap_cost::analytical::AnalyticalCostError; -use asap_cost::{ - kept_tumbling_window_chains, node_map, plan_cost, query_reach, tumbling_window_retention_ms, -}; - -type Ranking = PlanRanking; -use asap_types::workload::bounds::with_confirmed_weights; - -/// A cost per second for a rejection reason: three decimals, or three -/// significant digits in scientific notation when it is nonzero and below -/// 0.001 (e.g. "2.361e-8" under [`ResourceUnitCost::aws_fargate_us_east_1`]'s -/// dollars), so small costs do not all read "0.000". -fn display_cost(cost_per_second: f64) -> String { - if cost_per_second != 0.0 && cost_per_second.abs() < 1e-3 { - format!("{cost_per_second:.3e}") - } else { - format!("{cost_per_second:.3}") - } -} - -/// Select among whole plans: reject candidates that the executor cannot -/// run or that miss a query's accuracy target or latency bound (or, under -/// cost model 3, the cluster's capacity), measure the rest's resource -/// usage per second and choose one under the deployment's cost model -/// (`models.cost_inputs`): the cheapest under cost models 1 and 2, the -/// best by the cluster objective under cost model 3 (the first on ties). `entries[i]` is the workload -/// entry of `candidate.roots[i]`: its accuracy target, its latency bound and -/// how often it is evaluated. Every candidate is kept in the selection -/// ([`PlanSelector`] with `keep_every_candidate`). -#[cfg(any(test, feature = "reference-pricer"))] -pub(crate) fn select_among_plans( - cands: &[PhysicalWorkloadPlan], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: SelectionModels<'_>, -) -> Result { - let mut selector = PlanSelector::new(entries, data, models, true)?; - if selector.needs_calibration() { - let mut fit = CalibrationFit::default(); - for candidate in cands { - match selector.calibration_fit(candidate) { - // Rejected below when offered. - Err(ErrorSelectingPlan::Cost { - error: AnalyticalCostError::Overflow, - .. - }) => {} - fit_or_error => fit.add(fit_or_error?), - } - } - selector.set_calibration(fit); - } - // An unpriceable candidate is rejected, as the search rejects it - // (`cost_not_representable`), rather than failing the selection. - for candidate in cands { - match selector.offer(candidate) { - Err( - error @ ErrorSelectingPlan::Cost { - error: AnalyticalCostError::Overflow, - .. - }, - ) => selector.reject( - &candidate.id, - RejectionCause::CostNotRepresentable, - error.to_string(), - ), - result => result?, - } - } - selector.finish().map(|(selection, _)| selection) -} - -/// Whole plans offered one at a time, in candidate order -/// ([`select_among_plans`] over all of them at once). Each candidate is -/// checked and priced as it is offered; only the best so far is kept, -/// unless `keep_every_candidate`. Either way it selects what -/// [`select_among_plans`] selects over the same candidates. -pub struct PlanSelector<'a> { - entries: &'a [QueryWorkloadEntry], - data: &'a DataWorkload, - models: SelectionModels<'a>, - targets: Vec, - recurrence: Vec, - range_evaluations: Vec>, - ranking: Ranking, - keep_every_candidate: bool, - candidates_checked: u64, - costs: BTreeMap, - accepted: BTreeMap>, - rejected: Vec, - /// The valid candidates kept, in candidate order: every one, the best - /// so far, or the Pareto front so far. - kept: Vec, - /// The factor that calibrates the formulas' CPU time to the measured - /// table's ([`Self::set_calibration`]); 1 without a measured table. - pub(crate) formula_scale: f64, -} - -impl<'a> PlanSelector<'a> { - /// A selector for the workload `entries` over `data` under `models`. - /// Bad settings are an error here, before any candidate. - pub fn new( - entries: &'a [QueryWorkloadEntry], - data: &'a DataWorkload, - models: SelectionModels<'a>, - keep_every_candidate: bool, - ) -> Result { - models - .cost_inputs - .validate() - .map_err(ErrorSelectingPlan::InvalidSettings)?; - Ok(Self { - entries, - data, - models, - targets: accuracy_targets(entries), - recurrence: entries.iter().map(|e| e.recurrence.clone()).collect(), - range_evaluations: entries.iter().map(|e| e.range_evaluation).collect(), - ranking: Ranking::of(&models.cost_inputs.cost_model), - keep_every_candidate, - candidates_checked: 0, - costs: BTreeMap::new(), - accepted: BTreeMap::new(), - rejected: Vec::new(), - kept: Vec::new(), - formula_scale: models.cost_inputs.frozen_calibration_factor.unwrap_or(1.0), - }) - } - - /// Whether the deployment's measured cost table prices some operator, - /// so the formulas' CPU time is calibrated to it: one factor for every - /// candidate, fitted over the nodes the table prices in every candidate - /// that passes the checks before pricing. The caller then gives every - /// candidate to [`Self::calibration_fit`], adds up the fits and passes - /// the sum to [`Self::set_calibration`] before the first - /// [`Self::offer`]. - pub fn needs_calibration(&self) -> bool { - self.models.cost_inputs.frozen_calibration_factor.is_none() - && !self - .models - .cost_inputs - .measured_operation_costs - .operators - .is_empty() - } - - /// `candidate`'s part of the calibration fit: its measured nodes' CPU - /// time, as measured and as the formulas price it, when it passes the - /// checks before pricing; nothing otherwise. - pub fn calibration_fit( - &self, - candidate: &PhysicalWorkloadPlan, - ) -> Result { - if candidate.roots.len() != self.entries.len() { - return Err(ErrorSelectingPlan::EntryCount { - candidate: candidate.id.clone(), - roots: candidate.roots.len(), - entries: self.entries.len(), - }); - } - if accuracy_check(candidate, &self.targets, self.data, &self.models).is_err() { - return Ok(CalibrationFit::default()); - } - let nodes = node_map(&candidate.dag); - let kept_chains = kept_tumbling_window_chains(&candidate.dag); - let (_, fit) = plan_cost( - &candidate.dag, - &nodes, - &kept_chains, - &self.recurrence, - &self.range_evaluations, - self.data, - self.models.cost_inputs, - 1.0, - ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: candidate.id.clone(), - node, - error, - })?; - Ok(fit) - } - - /// Calibrate the formulas with `fit`, the sum of every candidate's - /// [`Self::calibration_fit`]. - pub fn set_calibration(&mut self, fit: CalibrationFit) { - self.formula_scale = self - .models - .cost_inputs - .frozen_calibration_factor - .unwrap_or_else(|| fit.factor()); - } - - /// Check and price `candidate`, the next in candidate order, and keep - /// it if it can still be selected. - #[cfg(any(test, feature = "reference-pricer"))] - pub fn offer(&mut self, candidate: &PhysicalWorkloadPlan) -> Result<(), ErrorSelectingPlan> { - if candidate.roots.len() != self.entries.len() { - return Err(ErrorSelectingPlan::EntryCount { - candidate: candidate.id.clone(), - roots: candidate.roots.len(), - entries: self.entries.len(), - }); - } - self.candidates_checked += 1; - let accepted_accuracy = - match accuracy_check(candidate, &self.targets, self.data, &self.models) { - Ok(accepted) => accepted, - Err(failed) => { - self.reject(&candidate.id, failed.cause, failed.reason); - return Ok(()); - } - }; - let nodes = node_map(&candidate.dag); - let kept_chains = kept_tumbling_window_chains(&candidate.dag); - let settings = self.models.cost_inputs; - let (cost, _) = plan_cost( - &candidate.dag, - &nodes, - &kept_chains, - &self.recurrence, - &self.range_evaluations, - self.data, - settings, - self.formula_scale, - ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: candidate.id.clone(), - node, - error, - })?; - let violation = - latency_violation(&cost, self.entries).or_else(|| match &settings.cost_model { - CostModel::ClusterCapacity(limits) => capacity_violation(&cost.usage, limits) - .map(|reason| (RejectionCause::OverClusterCapacity, reason)), - _ => None, - }); - if let Some((cause, reason)) = violation { - self.reject(&candidate.id, cause, reason); - return Ok(()); - } - if !self.keep_every_candidate { - // Ties keep the earlier candidate. - let selectable = self - .kept - .first() - .is_none_or(|best| self.ranking.better(&cost, &self.costs[&best.id])); - if !selectable { - return Ok(()); - } - } - if !self.keep_every_candidate { - for kept in self.kept.drain(..) { - self.costs.remove(&kept.id); - self.accepted.remove(&kept.id); - } - } - self.costs.insert(candidate.id.clone(), cost); - self.accepted - .insert(candidate.id.clone(), accepted_accuracy); - self.kept.push(candidate.clone()); - Ok(()) - } - - /// The selection over every candidate offered, and the selected - /// candidate as Stage 2 built it (the selection's plan is its roots - /// with the weight requirements Stage 3 confirmed made proofs). - pub fn finish( - mut self, - ) -> Result<(WholePlanSelection, PhysicalWorkloadPlan), ErrorSelectingPlan> { - let ranking = self.ranking; - let valid_costs: Vec<&PlanCost> = - self.kept.iter().map(|kept| &self.costs[&kept.id]).collect(); - let Some(best) = choose(ranking, &valid_costs) else { - return Err(ErrorSelectingPlan::NoValidCandidate(self.rejected)); - }; - let selected = self.kept[best].id.clone(); - for kept in &self.kept { - let id = &kept.id; - if *id != selected { - let (cause, reason) = lost_reason(ranking, id, &selected, &self.costs); - self.rejected.push(Rejection { - id: id.clone(), - cause, - reason, - }); - } - } - let built = self.kept.swap_remove(best); - let plan = with_confirmed_weights(&built.roots, self.data); - let tumbling_window_retention_ms = tumbling_window_retention_ms( - &built.dag, - &query_reach(&built.dag, &node_map(&built.dag)), - &self.range_evaluations, - ); - let selection = WholePlanSelection { - selected, - plan, - costs: self.costs, - accepted: self.accepted, - rejected: self.rejected, - tumbling_window_retention_ms, - candidates_checked: self.candidates_checked, - }; - Ok((selection, built)) - } - - /// Record that candidate `id` failed a check: every rejection when - /// keeping every candidate, otherwise the first of each cause. - fn reject(&mut self, id: &str, cause: RejectionCause, reason: String) { - if self.keep_every_candidate || !self.rejected.iter().any(|r| r.cause == cause) { - self.rejected.push(Rejection { - id: id.to_string(), - cause, - reason, - }); - } - } -} - -/// The index of the candidate `ranking` selects among `costs` (the first on -/// ties); `None` without candidates. -pub(crate) fn choose(ranking: Ranking, costs: &[&PlanCost]) -> Option { - (0..costs.len()).reduce( - |best, next| match ranking.better(costs[next], costs[best]) { - true => next, - false => best, - }, - ) -} - -/// Under cost model 3, the first limit of the cluster `usage` is over, as a -/// rejection reason. -fn capacity_violation(usage: &ResourceUsage, limits: &ClusterLimits) -> Option { - if usage.peak_memory_bytes > limits.memory_bytes { - return Some(format!( - "peak memory {} bytes is over the cluster's {} bytes", - usage.peak_memory_bytes, limits.memory_bytes - )); - } - if usage.disk_bytes > limits.disk_bytes { - return Some(format!( - "disk {} bytes is over the cluster's {} bytes", - usage.disk_bytes, limits.disk_bytes - )); - } - match limits.cpu_cores { - Some(cores) if usage.peak_cpu_cores > cores => Some(format!( - "{} CPU cores at peak is over the cluster's {cores}", - usage.peak_cpu_cores - )), - _ => None, - } -} - -/// Why valid candidate `id` was not `selected` under `ranking`, and the -/// reason with its numbers. -fn lost_reason( - ranking: Ranking, - id: &str, - selected: &str, - costs: &BTreeMap, -) -> (RejectionCause, String) { - let (cost, best) = (&costs[id], &costs[selected]); - if !ranking.better(best, cost) { - let tied = match ranking { - Ranking::Price => format!( - "{} {COST_UNIT}", - display_cost(cost.cost_per_second.unwrap_or_default()) - ), - _ => describe_usage(&cost.usage), - }; - return ( - RejectionCause::TiedLostOnEnumerationOrder, - format!( - "tied with the selected candidate {selected} at {tied}; lost on enumeration order" - ), - ); - } - match ranking { - Ranking::Price => ( - RejectionCause::Costlier, - format!( - "costlier: {} vs {} {COST_UNIT}", - display_cost(cost.cost_per_second.unwrap_or_default()), - display_cost(best.cost_per_second.unwrap_or_default()) - ), - ), - Ranking::Latency => ( - RejectionCause::Slower, - format!( - "slower: {} vs {}", - describe_usage(&cost.usage), - describe_usage(&best.usage) - ), - ), - Ranking::Memory => ( - RejectionCause::MoreMemory, - format!( - "more memory: {} vs {}", - describe_usage(&cost.usage), - describe_usage(&best.usage) - ), - ), - } -} - -/// E.g. `latency 126.0 ms, peak memory 9648000000 bytes, disk 480000000 bytes`. -fn describe_usage(usage: &ResourceUsage) -> String { - format!( - "latency {:.1} ms, peak memory {} bytes, disk {} bytes", - worst_latency_ms(usage), - usage.peak_memory_bytes, - usage.disk_bytes - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_cost::{QueryLatency, COST_SOURCE}; - - /// A candidate with the given latency, peak memory and disk under cost - /// model 3. - fn cluster_cost(latency_ms: f64, peak_memory_bytes: u64, disk_bytes: u64) -> PlanCost { - PlanCost { - usage: ResourceUsage { - peak_memory_bytes, - disk_bytes, - latency_per_query: vec![QueryLatency { - compaction_ms: 0.0, - query_ms: latency_ms, - }], - ..ResourceUsage::default() - }, - cost_per_second: None, - cost_unit: COST_UNIT, - source: COST_SOURCE, - per_node: BTreeMap::new(), - } - } - - /// Peak memory and disk are compared exactly: 100 bytes less at 124 GB - /// is less memory, not a tie. - #[test] - fn memory_differing_by_a_few_bytes_is_compared_exactly() { - let larger = cluster_cost(7.0, 124_000_000_100, 0); - let smaller = cluster_cost(7.0, 124_000_000_000, 0); - let costs = [&larger, &smaller]; - assert_eq!(choose(Ranking::Memory, &costs).unwrap(), 1); - assert_eq!(choose(Ranking::Latency, &costs).unwrap(), 1); - } -} diff --git a/crates/planner/Cargo.toml b/crates/planner/Cargo.toml index 80589f63a..6cf31bc74 100644 --- a/crates/planner/Cargo.toml +++ b/crates/planner/Cargo.toml @@ -31,21 +31,14 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" -[features] -# `test_support::query_time_reference`, for a runtime's correctness tests. -# It reads `asap-types`' test-only DAG helpers. -test-support = ["asap-types/test-support"] - [dev-dependencies] -# The reference pricer over fixed candidate plans. -asap-plan-selection = { path = "../plan-selection", features = ["reference-pricer"] } -# The tests compare installed plans with `test_support::query_time_reference`. -asap-planner = { path = ".", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } +asap-plan-selection = { path = "../plan-selection" } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } # Pass 1's target list (`per_computation_candidates`) for the tests. asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -# The deployment accuracy model tests implement `AccuracyModel`. asap-accuracy = { path = "../accuracy", features = ["test-support"] } +# Tests read a plan's roots as IR (`PhysicalASAPDAG::query_roots`). +asap-types = { path = "../types", features = ["test-support"] } # The SQL frontend plans through DataFusion, which is async; `e2e_plan` is # therefore async and its tests need a runtime. tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index ea836d674..df5f6669f 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -8,9 +8,9 @@ //! `retention_ms` of each materialized view and each query-time `Scan` on //! its node, and the query texts. //! Everything else a runtime needs is derived from the DAG -//! (`asap_types::ir::materialized_view`): the views to maintain, with what +//! (`asap_types::plan::materialized_view`): the views to maintain, with what //! each is refreshed from and when ([`InstallablePlan::materialized_views`]), -//! and how far back each raw-data source is kept for query-time scans +//! and how far back each raw-data source is retained for query-time scans //! ([`InstallablePlan::raw_data_retention`]). A runtime reads the file with //! [`InstallablePlan::from_json`]. Raw data is read only at the DAG's //! `Scan`s, by their `Source`, and a view by its `MaterializedView`: the @@ -23,26 +23,20 @@ //! costing and display) is not the runtime's concern: the installable DAG //! has every chain expanded into the tumbling windows the executor runs. -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::rc::Rc; +use std::collections::{HashMap, HashSet}; -use asap_cost::{NodePriceSource, OperatorKind}; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_plan_selection::{SearchReport, Selection}; -use asap_types::ir::materialized_view::{ +use asap_cost::{NodeCostSource, OperatorKind, PlanCost}; +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::plan::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, RawDataRetention, }; -use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; -use asap_types::ir::physical_export::{ + +use asap_types::ir::properties::ExecutionTiming; +use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, - PhysicalASAPNodeId, ReferencePlanError, -}; -use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; -use asap_types::ir::SchemaDerivationError; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, OperatorNode, - QueryRoot, + PhysicalASAPNodeId, }; + use asap_types::workload::QueryWorkloadEntry; use serde::{Deserialize, Serialize}; @@ -50,7 +44,7 @@ use crate::{e2e_plan, PlanError, UserInput}; /// The version of the plan-file format [`InstallablePlan`] serializes to: /// the DAG, its nodes' fields and the query texts. -pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 11; +pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 14; /// The selected plan, ready to install and execute: what /// [`plan_installable`] returns and a runtime writes as its plan file. @@ -69,7 +63,8 @@ pub struct InstallablePlan { pub query_texts: Vec, } -/// What Stage 3 recorded about the plan it selected, for reporting. +/// The selected plan's cost (`asap_cost::plan_cost` of the plan), for +/// reporting. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SelectedPlanReport { @@ -79,15 +74,6 @@ pub struct SelectedPlanReport { /// The resources Stage 3 predicts it uses, for a runtime to report /// against what it measures. pub predicted_usage: PredictedUsage, - /// How the search behind the selection ended (`Selection::search_report`). - pub search_report: SearchReport, - /// The workload queries (entry indices) whose output Stage 3 accepted - /// only because the accuracy model accepts it regardless of its - /// guarantee (e.g. `PlanningModels::ignore_accuracy_guarantees`): - /// the guarantee check would have rejected it. Empty when every - /// output's guarantee meets its target; omitted from JSON then. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub queries_accepted_without_guarantee: Vec, } /// What Stage 3 predicts the selected plan uses, per second of wall time @@ -104,7 +90,7 @@ pub struct PredictedUsage { pub average_memory_bytes: f64, /// The most memory live at once. pub peak_memory_bytes: u64, - /// Bytes kept on disk, raw data query-time scans read included. + /// Bytes retained on disk, raw data query-time scans read included. pub disk_bytes: u64, /// CPU cores to provision for the peak. pub peak_cpu_cores: u32, @@ -138,7 +124,7 @@ pub struct PredictedNodeUsage { /// 2 and 3. pub cost_per_second: Option, /// Whether its CPU came from the measured cost table or the formulas. - pub price_source: NodePriceSource, + pub cost_source: NodeCostSource, } /// One query's predicted latency per evaluation. @@ -156,14 +142,6 @@ pub struct PredictedQueryUsage { #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum InstallablePlanError { - /// Internal bug: the selected plan's timings do not hold, or it does - /// not export; or, for the query-time reference, bad input: the plan - /// cannot run at query time. - #[error("timing: {0}")] - Timing(#[from] ExecutionDataStateError), - /// Internal bug: a tumbling-window chain of the selected plan does not expand. - #[error("tumbling-window chain expansion: {0}")] - TumblingWindowChainExpansion(#[from] SchemaDerivationError), /// Bad input (a plan file): written for another plan-file format; `None` /// for a file without `wire_version`, written before version 11. #[error("unsupported installable plan wire version {0:?}")] @@ -188,10 +166,6 @@ pub enum InstallablePlanError { /// [`InstallableDagError`]). #[error("{0}")] Uninstallable(InstallableDagError), - /// Internal bug: Stage 3 records a tumbling-window retention for node `node_id` - /// of the selected plan, which is not one of its tumbling-window chains. - #[error("tumbling-window retention for node {node_id}, which is not a tumbling-window chain of the selected plan")] - TumblingWindowRetentionOfUnknownChain { node_id: PhysicalASAPNodeId }, /// Unsatisfiable request: the executor cannot refresh materialized view /// `node_id` from its inputs (`reason`); for a refresh group, `node_id` /// is its first view and `reason` names every view compiled with it. @@ -216,20 +190,10 @@ impl From for InstallablePlanError { } } -impl From for InstallablePlanError { - fn from(error: ReferencePlanError) -> Self { - match error { - ReferencePlanError::InvalidDag(e) => Self::InvalidDag(e), - ReferencePlanError::Timing(e) => Self::Timing(e), - } - } -} - /// Plan `input` for a deployment runtime: the query-language frontends and /// Stages 1-3 as [`e2e_plan`] runs them, returning the single selected DAG -/// ready to install, and beside it Stage 3's report of the selection -/// (`None` for a pass that selects without pricing), which is not part of -/// the plan file. Supported summaries the caller leaves unrestricted +/// ready to install, and beside it the plan's cost report, which is not +/// part of the plan file. Supported summaries the caller leaves unrestricted /// (`DeploymentCapabilities::supported_summaries` `None`) default to the /// families the Planner's executor runs; raw-data and query-time-result /// retention stay the caller's. Stage 3 does not compile candidates; @@ -250,27 +214,19 @@ pub async fn plan_installable( pass: input.pass, }; let output = e2e_plan(input).await?; - installable_plan_of( - &output.roots(), - output.selection.as_ref(), - &output.tumbling_window_retention_ms, - &entries, - ) - .map_err(PlanError::Installable) + let _ = (output, entries); + todo!("#1207 PR 6: install the plan Stage 3 built, with its plan_cost report") } -/// The installable plan of `roots`, the selected roots of a workload whose -/// entries are `entries` (in `QueryWorkload::entries()` order), with -/// Stage 3's report of its `selection` of them when there is one, and the -/// tumbling-window retention of the selected plan's kept chains -/// (`RealizedPlan::tumbling_window_retention_ms`, by node id in the DAG `roots` -/// export): what [`plan_installable`] returns after planning. A caller that -/// runs the planning stages itself (e.g. a developer tool) uses it to see -/// what a runtime would install. +/// The installable plan of `plan`, the plan Stage 3 built for a workload +/// whose entries are `entries` (in `QueryWorkload::entries()` order), with +/// the report of its `cost` (`asap_cost::plan_cost` of `plan`) when there is +/// one: what [`plan_installable`] returns after planning. A caller that runs +/// the planning stages itself (e.g. a developer tool) uses it to see what a +/// runtime would install. pub fn installable_plan_of( - roots: &[QueryRoot], - selection: Option<&Selection>, - tumbling_window_retention_ms: &BTreeMap, + plan: &PhysicalASAPDAG, + cost: Option<&PlanCost>, entries: &[QueryWorkloadEntry], ) -> Result<(InstallablePlan, Option), InstallablePlanError> { // Each query's longest range-evaluation span, which the views it @@ -280,14 +236,8 @@ pub fn installable_plan_of( .map(|entry| entry.range_evaluation.map_or(0, |range| range.span_ms)) .collect(); let query_texts: Vec = entries.iter().map(|entry| entry.query.0.clone()).collect(); - // Every node keeps the timing Stage 3 gave it. - let timed = - apply_materialization_timings_to_roots(roots, &MaterializationAssignment::default())?; - let expanded = tumbling_window_retention_of_expanded_tumbling_windows( - &timed, - tumbling_window_retention_ms, - )?; - let report = selection.map(|selection| report_of(selection, &expanded.installed_node_ids)); + let expanded = tumbling_window_retention_of_expanded_tumbling_windows(plan)?; + let report = cost.map(|cost| report_of(cost, &expanded.installed_node_ids)); let reader_span_ms_by_node = reader_span_ms_by_node(&expanded.dag, &range_spans_ms); let plan = InstallablePlan::of_dag( expanded.dag, @@ -315,7 +265,7 @@ impl InstallablePlan { Ok(self.dag.ingestion_refresh_groups()?) } - /// How far back each raw-data source is kept for query-time scans + /// How far back each raw-data source is retained for query-time scans /// ([`PhysicalASAPDAG::raw_data_retention`]): the raw data a runtime /// keeps, the raw-data counterpart of each view's retention. pub fn raw_data_retention(&self) -> Result, InstallablePlanError> { @@ -640,115 +590,27 @@ fn executor_refreshes( struct ExpandedWithRetention { dag: PhysicalASAPDAG, retention_ms_by_node: HashMap, - /// Each node id of the selected candidate's physical DAG (`timed` + /// Each node id of the selected candidate's physical DAG (its roots /// exported, Stage 3's ids) with the ids in `dag` that run it /// ([`PredictedNodeUsage::installed_node_ids`]). installed_node_ids: HashMap>, } -/// Expand the tumbling-window chains of `timed` and export it; each tumbling window takes the -/// retention Stage 2 gives its chain (`RealizedPlan::tumbling_window_retention_ms`, by -/// node id in the selected candidate's physical DAG, which is `timed` -/// exported). +/// The plan's DAG with its tumbling-window chains expanded; each tumbling +/// window takes its chain's retention (`retention_ms`, Stage 3's $h_v$). fn tumbling_window_retention_of_expanded_tumbling_windows( - timed: &[QueryRoot], - tumbling_window_retention_ms: &BTreeMap, + plan: &PhysicalASAPDAG, ) -> Result { - use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; - let selected = compile_physical_asap_workload_with_node_ids(timed)?; - // Reject unsupported raw-source shapes before expanding possibly thousands - // of tumbling windows. The complete expanded-plan validation remains authoritative. - selected.dag.check_raw_data_windows()?; - let mut retention_by_chain: Vec<(Rc, u64)> = Vec::new(); - for (&node_id, &retention_ms) in tumbling_window_retention_ms { - let chain = selected - .node_ids - .operator_node(node_id) - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .ok_or(InstallablePlanError::TumblingWindowRetentionOfUnknownChain { node_id })?; - retention_by_chain.push((Rc::clone(chain), retention_ms)); - } - let expansion = expand_tumbling_window_chains_in_workload(timed) - .map_err(InstallablePlanError::TumblingWindowChainExpansion)?; - let compiled = compile_physical_asap_workload_with_node_ids(&expansion.roots)?; - // Index physical identity once. Looking up every expanded tumbling window through - // the identity map's linear search makes this phase quadratic in tumbling windows. - let mut installed_ids_by_pointer = HashMap::with_capacity(compiled.dag.nodes.len()); - for node in &compiled.dag.nodes { - if let Some(operator) = compiled.node_ids.operator_node(node.id) { - installed_ids_by_pointer.insert(Rc::as_ptr(operator), node.id); - } - } - let installed_id = - |node: &Rc| installed_ids_by_pointer.get(&Rc::as_ptr(node)).copied(); - let mut retention_ms_by_node = HashMap::new(); - for expanded in &expansion.chains { - let Some((_, retention_ms)) = retention_by_chain - .iter() - .find(|(chain, _)| Rc::ptr_eq(chain, &expanded.chain)) - else { - continue; - }; - for tumbling_window in &expanded.tumbling_windows { - if let Some(node_id) = installed_id(tumbling_window) { - retention_ms_by_node.insert(node_id, *retention_ms); - } - } - } - // A chain runs as its tumbling windows; every other node as the node replacing it - // (itself when nothing below it changed). - let mut installed_node_ids = HashMap::new(); - for node in &selected.dag.nodes { - let Some(input) = selected.node_ids.operator_node(node.id) else { - continue; - }; - let ids: Vec = match expansion - .chains - .iter() - .find(|expanded| Rc::ptr_eq(&expanded.chain, input)) - { - Some(expanded) => { - let mut ids = Vec::with_capacity(expanded.tumbling_windows.len()); - for tumbling_window in &expanded.tumbling_windows { - if let Some(id) = installed_id(tumbling_window) { - ids.push(id); - } - } - ids - } - None => { - let replacement = expansion - .expanded_nodes - .iter() - .find(|(original, _)| Rc::ptr_eq(original, input)) - .map_or(input, |(_, replacement)| replacement); - installed_id(replacement).into_iter().collect() - } - }; - installed_node_ids.insert(node.id, ids); - } - Ok(ExpandedWithRetention { - dag: compiled.dag, - retention_ms_by_node, - installed_node_ids, - }) + let _ = plan; + todo!("#1207 PR 6: expand the plan's tumbling-window chains") } -/// Stage 3's record of its selected plan. +/// The report of the selected plan's `cost`. fn report_of( - selection: &Selection, + cost: &PlanCost, installed_node_ids: &HashMap>, ) -> SelectedPlanReport { - let cost = &selection.cost; - let queries_accepted_without_guarantee = selection - .accepted_accuracy_per_query - .iter() - .enumerate() - .filter(|(_, accepted)| accepted.guarantee_check_rejection.is_some()) - .map(|(query, _)| query) - .collect(); SelectedPlanReport { - queries_accepted_without_guarantee, cost_per_second: cost.cost_per_second, predicted_usage: PredictedUsage { cpu_core_seconds_per_second: cost @@ -771,7 +633,7 @@ fn report_of( timing: node.timing, cpu_core_seconds_per_second: node.cpu_core_seconds_per_second, cost_per_second: node.cost_per_second, - price_source: node.price_source, + cost_source: node.cost_source, }) .collect(), per_query: cost @@ -784,16 +646,13 @@ fn report_of( }) .collect(), }, - search_report: selection.search_report.clone(), } } #[cfg(test)] mod tests { use super::*; - use asap_types::ir::schema::{SketchAlgorithm, StatisticKind, SupportedSummary}; - use asap_types::ir::{NonASAPOp, Operator}; - use asap_types::physical::Materialization; + use asap_types::summary_type::{SketchAlgorithm, StatisticKind, SupportedSummary}; thread_local! { /// The plan nodes handed to the executor's compiler by the views' @@ -802,7 +661,7 @@ mod tests { pub(super) static COMPILED_NODES: std::cell::Cell = const { std::cell::Cell::new(0) }; } - /// A p99 panel over a 300 s window every second, with no raw data kept: + /// A p99 panel over a 300 s window every second, with no raw data retained: /// one DDSketch per series in each of 300 one-second tumbling windows built at /// ingestion, about 300 materialized views in a DAG of about 900 nodes. /// Each refresh (here one refresh group of all the tumbling windows) is compiled @@ -822,7 +681,7 @@ mod tests { "predictability": {"predictable": {"known_at": null}}}]}, "data_workload": {"arrival": {"ingesting": {"rows_per_second": 10000.0}}, "sources": {"data": {"series_count": 1000, "metric_type": "gauge", - "sample_interval": 100, "values_non_negative": true}}}}"#, + "sample_interval": 100, "column_signs": {"value": "non_negative"}}}}}"#, ) .unwrap(); let capabilities = DeploymentCapabilities { @@ -859,200 +718,6 @@ mod tests { ); } - /// `TimeShift(10 min) <- TimeRange(5 min) <- Scan`, all at query time: - /// the shift above the range moves its window, which a window of a - /// fixed offset below the range cannot say, so it is refused; without - /// the shift the window is listed. - #[test] - fn a_time_shift_above_the_range_is_refused() { - use asap_types::ir::operator::operator_properties::TimeShift; - use asap_types::ir::operator::Source; - use asap_types::ir::physical_export::compile_physical_asap_dag; - use asap_types::ir::schema::{DataType, Field, Schema}; - use asap_types::ir::{ - apply_materialization_timings, NodeToTimedNodeCache, OperatorNode, TimeRangeKind, - }; - use std::time::Duration; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency_ms".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: scan, - })) - .unwrap(); - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 600_000, - at: None, - }, - child: std::rc::Rc::clone(&range), - })) - .unwrap(); - let export = |root| { - let timed = apply_materialization_timings( - &root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() - }; - let install = |dag| { - InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) - }; - match install(export(shifted)) { - Err(InstallablePlanError::Uninstallable( - InstallableDagError::UnsupportedRawDataWindow { reason, .. }, - )) => { - assert_eq!(reason, "a subquery or a TimeShift above the range") - } - other => panic!("{other:?}"), - } - let plan = install(export(range)).unwrap(); - assert!(plan.materialized_views().unwrap().is_empty()); - assert_eq!( - plan.raw_data_retention().unwrap(), - [RawDataRetention::new( - Source::TimeSeries { - metric: "latency_ms".into() - }, - Some(300_000) - )] - ); - } - - // A summary stored at ingestion time over a window ending at the - // evaluation and one ending a minute before it cannot be built in one - // ingestion run, so the plan is refused, not built wrong. - #[test] - fn an_ingestion_build_of_windows_at_different_offsets_is_refused() { - use asap_types::ir::operator::operator_properties::TimeShift; - use asap_types::ir::operator::{Reduction, Source}; - use asap_types::ir::physical_export::compile_physical_asap_dag; - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, SummaryUpdate, - }; - use asap_types::ir::{ - apply_materialization_timings, NodeToTimedNodeCache, OperatorNode, TimeRangeKind, - }; - use asap_types::physical::Medium; - use std::time::Duration; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency_ms".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - // A one-minute window ending `offset_ms` before the evaluation. - let window = |offset_ms: i64| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms, - at: None, - }, - child: Rc::clone(&scan), - })) - .unwrap(); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(60), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap() - }; - // One sum over both windows' rows. - let both = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Concat { - children: vec![window(0), window(60_000)], - })) - .unwrap(); - let merged = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: both, - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&merged, Materialization::MatAtIngestion(Medium::Memory)); - let timed = - apply_materialization_timings(&merged, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - match InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) { - Err(InstallablePlanError::Uninstallable( - InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { - end_offsets_ms, - .. - }, - )) => assert_eq!(end_offsets_ms, [0, 60_000]), - other => panic!("{other:?}"), - } - // A merge stored at ingestion time over two tumbling windows stored at - // ingestion time, one a minute older: an ingestion run would read - // both tumbling windows for the window ending at its end, so it is refused too. - let tumbling_window = |offset_ms: i64| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: window(offset_ms), - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - })) - .unwrap() - }; - let tumbling_windows = [tumbling_window(0), tumbling_window(60_000)]; - let merge = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: tumbling_windows.to_vec(), - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - for node in tumbling_windows.iter().chain([&merge]) { - assignment.set(node, Materialization::MatAtIngestion(Medium::Memory)); - } - let timed = - apply_materialization_timings(&merge, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - match InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) { - Err(InstallablePlanError::Uninstallable( - InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { - end_offsets_ms, - .. - }, - )) => assert_eq!(end_offsets_ms, [0, 60_000]), - other => panic!("{other:?}"), - } - } - /// Unrestricted summaries become the executor's families; a caller's /// own list and its retention flags are kept. #[test] @@ -1070,7 +735,7 @@ mod tests { assert!(!defaulted.supports(&SketchAlgorithm::CountSketch, StatisticKind::ItemFrequency)); let kll_only = DeploymentCapabilities { supported_summaries: Some(vec![SupportedSummary { - family: SketchAlgorithm::Kll, + summary_type: SketchAlgorithm::Kll, readouts: vec![StatisticKind::Quantile], }]), ..DeploymentCapabilities::default() diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index ffefd3f12..194147021 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -6,10 +6,10 @@ //! ``` //! //! The default pass, [`Stage1ToStage3Pass`], orchestrates the stages -//! ([`run_planning`]): Stage 0, Pass 1, Pass 2 over every tumbling-window width, -//! Stage 2 (`generate_physical_asap_memo_candidates`) and Stage 3 -//! (`asap_plan_selection::select_plan`), and builds the selected plan through -//! Stage 2's realizer. The stages never call each other. +//! ([`run_planning`]): Stage 0, Pass 1, Pass 2, Stage 2 +//! (`generate_physical_asap_memo_candidates`) and Stage 3 +//! (`asap_plan_selection::select_plan`), which builds the selected plan. +//! The stages never call each other. //! //! [`e2e_plan`] runs both steps. [`plan_installable`] runs them for a //! deployment runtime and returns the single selected DAG ready to install @@ -29,11 +29,11 @@ mod pass; mod run; pub use pass::{ optimize, OptimizationInput, OptimizationInputError, OptimizationPass, OptimizeError, - PassNameConflict, PassRegistry, PlanOutput, PlanningModels, QueryPlan, + PassNameConflict, PassRegistry, PlanOutput, PlanningModels, }; pub use run::{ - run_planning, selected_chosen_candidates, PlanSearchSettings, PlanningReport, PlanningRun, - SelectionError, Stage1ToStage3Pass, + run_planning, PlanSearchSettings, PlanningReport, PlanningRun, SelectionError, + Stage1ToStage3Pass, }; mod installable; @@ -43,52 +43,26 @@ pub use installable::{ INSTALLABLE_PLAN_WIRE_VERSION, }; -/// Helpers for a runtime's correctness tests (feature `test-support`). -#[cfg(any(test, feature = "test-support"))] -pub mod test_support { - use std::collections::HashMap; - - use crate::{InstallablePlan, InstallablePlanError}; - - /// For correctness tests only, never for installing: `plan` with - /// nothing materialized, the same operators, summaries and parameters - /// with every node at query time, and the same query texts. A runtime's - /// test compares its installed plan's answers with the executor running - /// this one over raw data samples. Validates `plan` first. - pub fn query_time_reference( - plan: &InstallablePlan, - ) -> Result { - plan.validate()?; - let dag = plan.dag.with_every_node_at_query_time()?; - InstallablePlan::of_dag( - dag, - plan.query_texts.clone(), - &HashMap::new(), - &HashMap::new(), - ) - } -} - // What a deployment runtime needs to build a `UserInput` and read an // `InstallablePlan`, so it depends on this crate and `asap-executor` alone. pub use asap_cost::{ ClusterLimits, ClusterObjective, CostInputs, CostModel, CostTableProvenance, - MeasuredOperationCost, MeasuredOperationCosts, NodePriceSource, OperatorCost, OperatorKind, + MeasuredOperationCost, MeasuredOperationCosts, NodeCostSource, OperatorCost, OperatorKind, ResourceSpeed, ResourceUnitCost, SummaryOperation, }; pub use asap_frontend_sql::SqlCatalog; -pub use asap_logical_asap_optimization::{DeploymentCapabilities, SupportedSummary}; -pub use asap_plan_selection::{ - AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, SearchReport, SearchStatus, Selection, +pub use asap_plan_selection::{AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection}; +pub use asap_types::deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; +pub use asap_types::ir::properties::summary_coverage::{ + ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, }; -pub use asap_types::ir::materialized_view::{ +pub use asap_types::ir::properties::ExecutionTiming; +pub use asap_types::plan::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, MaterializedViewInputs, MaterializedViewRefresh, RawDataRetention, RawDataWindow, RefreshWindow, ViewCoverage, }; -pub use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; -pub use asap_types::ir::properties::summary_coverage::{ColumnIdentity, Constraint, SelectionBox}; -pub use asap_types::ir::properties::ExecutionTiming; -pub use asap_types::ir::schema::{SketchAlgorithm, SketchParams, StatisticKind}; +pub use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +pub use asap_types::summary_type::{SketchAlgorithm, SketchParams, StatisticKind}; pub use asap_types::workload::PlanningWorkload; // ── Input ──────────────────────────────────────────────────────────────── @@ -250,17 +224,12 @@ async fn lower(input: &UserInput<'_>) -> Result, }; let mut lowered = Vec::new(); for (index, entry) in input.workload.query_workload.entries().enumerate() { - let expr = lower_sql_dialect( - &entry.query.0, - catalog, - dialect.clone(), - entry.requirements.accuracy.clone(), - ) - .await - .map_err(|source| PlanError::Lowering { - entry_index: Some(index), - source: LoweringError::Sql(source), - })?; + let expr = lower_sql_dialect(&entry.query.0, catalog, dialect.clone()) + .await + .map_err(|source| PlanError::Lowering { + entry_index: Some(index), + source: LoweringError::Sql(source), + })?; lowered.push(expr.into()); } Ok(lowered) diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index 9d699cfc9..8b4d2528d 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -14,31 +14,17 @@ //! downstream consumers rely on. use std::collections::BTreeMap; -use std::rc::Rc; - -use asap_types::ir::physical_export::compile_physical_asap_workload; -use asap_types::ir::properties::timing::{ - apply_materialization_timings_to_roots, MaterializationAssignment, -}; -use asap_types::ir::properties::ExecutionDataStateError; -use asap_types::ir::OperatorNode; + use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; use crate::run::{SelectionError, Stage1ToStage3Pass}; -use asap_accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, IgnoreAccuracyGuarantees, - NoAccuracyEvidence, -}; +use asap_accuracy::{AccuracyEvidence, NoMeasuredGuarantees}; use asap_cost::CostInputs; -use asap_logical_asap_optimization::DeploymentCapabilities; use asap_logical_asap_optimization::PerComputationCandidateError; -use asap_plan_selection::{AccuracyCheckInputs, Selection}; -use asap_types::ir::physical_export::PhysicalASAPNodeId; +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPDAGValidationError}; -static BUILTIN_ACCURACY_MODEL: BuiltinAccuracyModel = BuiltinAccuracyModel; -static IGNORE_ACCURACY_GUARANTEES: IgnoreAccuracyGuarantees = IgnoreAccuracyGuarantees; -static NO_ACCURACY_EVIDENCE: NoAccuracyEvidence = NoAccuracyEvidence; static DEFAULT_COST_INPUTS: std::sync::LazyLock = std::sync::LazyLock::new(CostInputs::default); static DEFAULT_DEPLOYMENT: std::sync::LazyLock = @@ -50,21 +36,15 @@ static DEFAULT_DEPLOYMENT: std::sync::LazyLock = /// a built-in default, evidence about a particular deployment cannot. /// /// The facade splits them by stage: Stage 1 reads the supported summaries -/// of `deployment_capabilities`, Stage 2 its retention, and Stage 3 -/// (plan selection) checks each candidate's guarantee with `accuracy` and -/// `evidence` ([`AccuracyCheckInputs`]) and compares candidates under -/// `cost_inputs` ([`CostInputs`]). +/// of `deployment_capabilities`, Pass 2 the accuracy `evidence`, Stage 2 the +/// deployment's retention, and Stage 3 (plan selection) compares candidates +/// under `cost_inputs` ([`CostInputs`]). #[derive(Clone, Copy)] #[non_exhaustive] pub struct PlanningModels<'a> { - /// Decides whether a candidate's accuracy guarantee meets its query's - /// target, and whether the deployment accepts an output regardless of - /// its guarantee; [`BuiltinAccuracyModel`] by default. - /// `ignore_accuracy_guarantees` overrides it. - pub accuracy: &'a dyn AccuracyModel, - /// Deployment facts the guarantees need (operand domains, observed - /// guarantees); [`NoAccuracyEvidence`] by default. - pub evidence: &'a dyn AccuracyEvidenceProvider, + /// The deployment's measured accuracy, which Pass 2 reads before each + /// summary's theoretical guarantee; none by default. + pub evidence: &'a dyn AccuracyEvidence, /// The deployment's cost inputs: the cost model (default /// cost model 1 with illustrative unit costs), the resource speed and /// the non-repeating query horizon. @@ -74,25 +54,18 @@ pub struct PlanningModels<'a> { /// whether it keeps query-time results across evaluations (Stage 2); /// default [`DeploymentCapabilities::default`]. pub deployment_capabilities: &'a DeploymentCapabilities, - /// Accept every query output regardless of its accuracy guarantee: the - /// facade passes Stage 3 [`IgnoreAccuracyGuarantees`] in place of - /// `accuracy`. For comparing candidates on cost alone, e.g. a - /// benchmark of sketch plans with no analytical guarantee; - /// executability checks still apply, and each output accepted only - /// because of it is marked in the report - /// (`AcceptedAccuracy::guarantee_check_rejection`). Default `false`. + /// Do not enforce accuracy: the facade gives every query "no accuracy + /// target" before Pass 2, so no readout gets a required accuracy and + /// every θ connects (a top-k's heap still holds k). For comparing + /// candidates on cost alone, e.g. a benchmark of sketch plans with no + /// analytical guarantee. Default `false`. pub ignore_accuracy_guarantees: bool, } impl<'a> PlanningModels<'a> { - /// These accuracy and evidence models with the default plan selection - /// settings. - pub fn new( - accuracy: &'a dyn AccuracyModel, - evidence: &'a dyn AccuracyEvidenceProvider, - ) -> Self { + /// This evidence with the default plan selection settings. + pub fn new(evidence: &'a dyn AccuracyEvidence) -> Self { Self { - accuracy, evidence, cost_inputs: &DEFAULT_COST_INPUTS, deployment_capabilities: &DEFAULT_DEPLOYMENT, @@ -100,43 +73,19 @@ impl<'a> PlanningModels<'a> { } } - /// The built-in models. + /// The built-in models: no accuracy evidence. pub fn builtin() -> PlanningModels<'static> { - PlanningModels { - accuracy: &BUILTIN_ACCURACY_MODEL, - evidence: &NO_ACCURACY_EVIDENCE, - cost_inputs: &DEFAULT_COST_INPUTS, - deployment_capabilities: &DEFAULT_DEPLOYMENT, - ignore_accuracy_guarantees: false, - } - } - - /// Stage 3's accuracy check inputs: `evidence`, and `accuracy`, or - /// [`IgnoreAccuracyGuarantees`] with `ignore_accuracy_guarantees`. - pub fn accuracy_check_inputs(&self) -> AccuracyCheckInputs<'a> { - AccuracyCheckInputs { - model: if self.ignore_accuracy_guarantees { - &IGNORE_ACCURACY_GUARANTEES - } else { - self.accuracy - }, - evidence: self.evidence, - } + PlanningModels::new(&NoMeasuredGuarantees) } - /// These models, accepting every query output regardless of its - /// accuracy guarantee when `ignore` (`ignore_accuracy_guarantees`). + /// These models, enforcing no accuracy target when `ignore` + /// (`ignore_accuracy_guarantees`). pub fn with_ignore_accuracy_guarantees(mut self, ignore: bool) -> Self { self.ignore_accuracy_guarantees = ignore; self } - pub fn with_accuracy(mut self, accuracy: &'a dyn AccuracyModel) -> Self { - self.accuracy = accuracy; - self - } - - pub fn with_evidence(mut self, evidence: &'a dyn AccuracyEvidenceProvider) -> Self { + pub fn with_evidence(mut self, evidence: &'a dyn AccuracyEvidence) -> Self { self.evidence = evidence; self } @@ -187,149 +136,19 @@ pub enum OptimizationInputError { // ── Output ─────────────────────────────────────────────────────────────── -/// One query's selected post-ASAP DAG, or one operator sub-DAG a standalone -/// scalar query reads. The DAG is `root`. -#[derive(Debug, Clone)] -pub struct QueryPlan { - /// Index into `QueryWorkload::entries()`. - pub entry_index: usize, - pub root: Rc, -} - -/// One multi-root workload DAG with query bindings in entry order; -/// `check_contract` enforces that. -/// -/// Plans are not deduplicated across entries: a summary state that several -/// queries share appears in each of their plans as the same `Rc`, so a -/// consumer that deploys or costs the workload must dedupe by `Rc::ptr_eq`. +/// The selected plan: one `PhysicalASAPDAG`, the only plan structure, in +/// memory and, serialized, in the plan file installed on the executor. Its +/// `roots` answer the operator queries and its `scalar_roots` the standalone +/// scalar queries, in entry order; `check_contract` enforces that. #[derive(Debug, Clone)] #[non_exhaustive] pub struct PlanOutput { - pub plans: Vec, - /// Standalone scalar queries, keyed by position in the batch, which is - /// the entry index because `check_contract` requires dense entry - /// indices. Each operator reference is the root of one of - /// `scalar_plans`. - pub scalar_roots: Vec<(usize, asap_types::ir::ScalarExpr)>, - /// One plan per distinct operator sub-DAG a scalar root reads, keyed by - /// that scalar's entry, in `scalar_roots` order. - pub scalar_plans: Vec, - /// How plan selection chose the plan, when the pass selects among priced - /// candidates. - pub selection: Option, - /// For each tumbling-window chain of the selected plan kept beyond one - /// evaluation, by node id in its physical DAG, how long its completed - /// tumbling windows are kept ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)); - /// empty for a pass that does not set it. - pub tumbling_window_retention_ms: BTreeMap, + pub plan: PhysicalASAPDAG, } impl PlanOutput { - pub fn new(plans: Vec) -> Self { - Self { - plans, - scalar_roots: Vec::new(), - scalar_plans: Vec::new(), - selection: None, - tumbling_window_retention_ms: BTreeMap::new(), - } - } - - /// Entry indices in output order. - pub fn entry_indices(&self) -> Vec { - let mut indices: Vec<_> = self - .plans - .iter() - .map(|p| p.entry_index) - .chain(self.scalar_roots.iter().map(|(i, _)| *i)) - .collect(); - if !self.scalar_roots.is_empty() { - indices.sort_unstable(); - } - indices - } - - /// All query roots in workload order, including standalone scalars. - pub fn roots(&self) -> Vec { - let mut roots: Vec<_> = self - .plans - .iter() - .map(|p| { - ( - p.entry_index, - asap_types::ir::QueryRoot::Operator(Rc::clone(&p.root)), - ) - }) - .chain( - self.scalar_roots - .iter() - .map(|(i, expr)| (*i, asap_types::ir::QueryRoot::Scalar(expr.clone()))), - ) - .collect(); - roots.sort_by_key(|(i, _)| *i); - roots.into_iter().map(|(_, root)| root).collect() - } - - /// The selected operator roots. Use `roots()` to include scalar queries. - pub fn operator_roots(&self) -> Vec> { - self.plans.iter().map(|p| Rc::clone(&p.root)).collect() - } - - /// Unique operators in the entire workload DAG, including scalar-plan dependencies. - /// Several query roots can reach the same operator; it is returned once. - pub fn operators(&self) -> Vec> { - let mut seen = std::collections::HashSet::new(); - let mut nodes = Vec::new(); - for root in self.roots() { - let inputs = match root { - asap_types::ir::QueryRoot::Operator(node) => vec![node], - asap_types::ir::QueryRoot::Scalar(expr) => { - expr.operator_refs().into_iter().cloned().collect() - } - }; - for input in inputs { - for node in OperatorNode::reachable(&input) { - if seen.insert(Rc::as_ptr(&node)) { - nodes.push(node); - } - } - } - } - nodes - } - - /// The workload as one physical ASAP DAG, with shared sub-DAGs exported - /// once. A node keeps the timing and materialization its pass gave it: - /// for [`Stage1ToStage3Pass`] that is Stage 3's selected plan as - /// selected (e.g. ingestion-time tumbling windows), the document a deployment - /// runtime executes. Only a node the pass left untimed takes the default - /// [`MaterializationAssignment`] (not materialized, computed when its - /// consumer runs). Operator queries become its `roots` and standalone scalar - /// queries its `scalar_roots`, keyed by position in the batch. That - /// position is the entry index, because `check_contract` requires dense - /// entry indices. - /// - /// An empty workload is an error - /// ([`PhysicalASAPDAGValidationError::NoRoots`](asap_types::ir::physical_export::PhysicalASAPDAGValidationError::NoRoots)). - /// A workload of only literal scalars yields a DAG with no nodes: just - /// its `scalar_roots`. - pub fn physical_asap_dag( - &self, - ) -> Result { - // A node several roots share by `Rc` is timed and exported once. - let timed = apply_materialization_timings_to_roots( - &self.roots(), - &MaterializationAssignment::default(), - )?; - compile_physical_asap_workload(&timed) - } - - pub fn len(&self) -> usize { - self.plans.len() + self.scalar_roots.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 + pub fn new(plan: PhysicalASAPDAG) -> Self { + Self { plan } } } @@ -346,6 +165,12 @@ pub enum OptimizeError { /// defect in the pass, not in its input. #[error("pass `{pass}` violated the output contract: {detail}")] ContractViolation { pass: &'static str, detail: String }, + /// The pass returned an invalid DAG: a defect in the pass. + #[error("pass `{pass}` returned an invalid plan: {error}")] + InvalidPlan { + pass: &'static str, + error: PhysicalASAPDAGValidationError, + }, } // ── The pass ───────────────────────────────────────────────────────────── @@ -364,9 +189,9 @@ pub trait OptimizationPass { /// Run `pass` over `input`: validate once, then check the output contract. /// /// Prefer this over calling [`OptimizationPass::optimize`] directly. A pass is -/// third-party code, but `ASAPQuery-backend` and `ASAPCollector` read every -/// pass's output against the same contract; checking here means a defective -/// pass fails at the boundary instead of corrupting a deployment. +/// third-party code, but every deployment runtime reads every pass's output +/// against the same contract; checking here means a defective pass fails at +/// the boundary instead of corrupting a deployment. pub fn optimize( pass: &dyn OptimizationPass, input: OptimizationInput<'_>, @@ -379,53 +204,26 @@ pub fn optimize( Ok(output) } -/// Structural checks only — that every query is accounted for, in order. -/// Whether the pass chose *well* is not checked. +/// Structural checks only — that every query is answered once, by an +/// operator root or the scalar root keyed with its entry index, and that +/// the DAG is valid. Whether the pass chose *well* is not checked. fn check_contract( output: &PlanOutput, pass: &'static str, expected_len: usize, ) -> Result<(), OptimizeError> { - let violation = |detail: String| OptimizeError::ContractViolation { pass, detail }; - - if output.len() != expected_len { - return Err(violation(format!( - "{} plan(s) for {expected_len} workload entry/entries", - output.len() - ))); - } - for (position, entry_index) in output.entry_indices().into_iter().enumerate() { - if entry_index != position { - return Err(violation(format!( - "plan at position {position} carries entry_index {entry_index}" - ))); - } - } - // `scalar_plans` holds one plan per distinct reference of each scalar - // root, in `scalar_roots` order, keyed by that scalar's entry. - let mut scalar_plans = output.scalar_plans.iter(); - for (entry_index, expr) in &output.scalar_roots { - let mut refs: Vec<&Rc> = Vec::new(); - for node in expr.operator_refs() { - if !refs.iter().any(|r| Rc::ptr_eq(r, node)) { - refs.push(node); - } - } - for node in refs { - match scalar_plans.next() { - Some(plan) if plan.entry_index == *entry_index && Rc::ptr_eq(&plan.root, node) => {} - _ => { - return Err(violation(format!( - "scalar root {entry_index} reads an operator without its scalar plan" - ))) - } - } - } - } - if scalar_plans.next().is_some() { - return Err(violation( - "a scalar plan is read by no scalar root".to_string(), - )); + let plan = &output.plan; + let answered = plan.roots.len() + plan.scalar_roots.len(); + if answered != expected_len { + return Err(OptimizeError::ContractViolation { + pass, + detail: format!("{answered} plan root(s) for {expected_len} workload entry/entries"), + }); + } + // An empty workload's plan has no roots, which `validate` rejects. + if expected_len > 0 { + plan.validate() + .map_err(|error| OptimizeError::InvalidPlan { pass, error })?; } Ok(()) } @@ -496,159 +294,60 @@ mod tests { struct Stub(&'static str); + fn empty_plan() -> PlanOutput { + PlanOutput::new(PhysicalASAPDAG { + nodes: vec![], + edges: vec![], + roots: vec![], + scalar_roots: vec![], + }) + } + impl OptimizationPass for Stub { fn name(&self) -> &'static str { self.0 } fn optimize(&self, _input: OptimizationInput<'_>) -> Result { - Ok(PlanOutput::new(Vec::new())) + Ok(empty_plan()) } } - /// Dropping a query is rejected: the output is positionally aligned with - /// the workload's entries, so a short vector is not a partial result. + /// Dropping a query is rejected: the plan's roots answer the workload's + /// entries in order, so a plan with fewer roots is not a partial result. #[test] fn contract_rejects_a_plan_count_that_does_not_cover_every_entry() { - let output = PlanOutput::new(Vec::new()); - let err = check_contract(&output, "stub", 2).unwrap_err(); + let err = check_contract(&empty_plan(), "stub", 2).unwrap_err(); assert!(matches!( err, OptimizeError::ContractViolation { pass: "stub", .. } )); } - /// A scalar root's operator reference must be the root of its scalar - /// plan; a scalar root without one is rejected. + /// A scalar root keyed by an entry index the workload does not have is + /// rejected as an invalid plan. #[test] - fn contract_rejects_a_scalar_reference_without_its_plan() { - use asap_types::ir::operator::non_asap::NonASAPOp; - use asap_types::ir::operator::operator_properties::Source; - use asap_types::ir::schema::{DataType, Field, Schema}; - use asap_types::ir::{Operator, ScalarExpr}; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![Field::plain("ts", DataType::Timestamp, false)], - 0, - vec![], - ), - })) - .unwrap(); - let mut output = PlanOutput::new(Vec::new()); + fn contract_rejects_a_scalar_root_keyed_beyond_the_entries() { + use asap_types::ir::scalar::ScalarValue; + use asap_types::ir::ScalarExpr; + let mut output = empty_plan(); output + .plan .scalar_roots - .push((0, ScalarExpr::PromqlScalarFromVector(scan))); + .push((1, ScalarExpr::Literal(ScalarValue::Float64(1.0)))); let err = check_contract(&output, "stub", 1).unwrap_err(); assert!(matches!( err, - OptimizeError::ContractViolation { pass: "stub", .. } + OptimizeError::InvalidPlan { + pass: "stub", + error: PhysicalASAPDAGValidationError::ScalarRootPosition(1) + } )); } - /// A scalar plan read from `root`, keyed by `entry_index`. - fn scalar_plan(entry_index: usize, root: &Rc) -> QueryPlan { - QueryPlan { - entry_index, - root: Rc::clone(root), - } - } - - /// `scalar_plans` holds one plan per distinct reference of each scalar - /// root, in order and keyed by that root's entry: a plan out of order, - /// under another entry, or read by no scalar root is rejected. - #[test] - fn contract_checks_scalar_plan_order_entry_and_count() { - use asap_types::ir::operator::non_asap::NonASAPOp; - use asap_types::ir::operator::operator_properties::Source; - use asap_types::ir::scalar::ArithmeticOpKind; - use asap_types::ir::schema::{DataType, Field, Schema}; - use asap_types::ir::{ExprSemantics, Operator, ScalarExpr}; - let series = |metric: &str| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![Field::plain("ts", DataType::Timestamp, false)], - 0, - vec![], - ), - })) - .unwrap() - }; - let (a, b) = (series("a"), series("b")); - let read = |node: &Rc| ScalarExpr::PromqlScalarFromVector(Rc::clone(node)); - let combine = |left, right| ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Sub, - left: Box::new(left), - right: Box::new(right), - semantics: ExprSemantics::Promql, - }; - let with_plans = |plans: Vec| { - let mut output = PlanOutput::new(Vec::new()); - // `a - b`, then `a - a`: one plan per distinct reference. - output.scalar_roots = vec![ - (0, combine(read(&a), read(&b))), - (1, combine(read(&a), read(&a))), - ]; - output.scalar_plans = plans; - check_contract(&output, "stub", 2) - }; - with_plans(vec![ - scalar_plan(0, &a), - scalar_plan(0, &b), - scalar_plan(1, &a), - ]) - .unwrap(); - for plans in [ - // Wrong order. - vec![scalar_plan(0, &b), scalar_plan(0, &a), scalar_plan(1, &a)], - // Wrong entry key. - vec![scalar_plan(0, &a), scalar_plan(1, &b), scalar_plan(1, &a)], - // An extra plan. - vec![ - scalar_plan(0, &a), - scalar_plan(0, &b), - scalar_plan(1, &a), - scalar_plan(1, &a), - ], - ] { - assert!(matches!( - with_plans(plans), - Err(OptimizeError::ContractViolation { pass: "stub", .. }) - )); - } - } - /// The matching count passes. #[test] fn contract_accepts_an_empty_workload() { - let output = PlanOutput::new(Vec::new()); - assert!(check_contract(&output, "stub", 0).is_ok()); - } - - /// An empty workload has no DAG: the export's `NoRoots`, wrapped in - /// `InvalidPhysicalDAG`. A workload of only literal scalars exports a DAG - /// with no nodes, just its `scalar_roots`. - #[test] - fn physical_asap_dag_of_an_empty_and_a_literal_only_workload() { - use asap_types::ir::physical_export::PhysicalASAPDAGValidationError; - use asap_types::ir::ScalarExpr; - assert_eq!( - PlanOutput::new(Vec::new()).physical_asap_dag().err(), - Some(ExecutionDataStateError::InvalidPhysicalDAG( - PhysicalASAPDAGValidationError::NoRoots - )) - ); - let mut literal = PlanOutput::new(Vec::new()); - literal.scalar_roots = vec![(0, ScalarExpr::literal_f64(1.0))]; - let dag = literal.physical_asap_dag().unwrap(); - assert!(dag.nodes.is_empty() && dag.roots.is_empty()); - let [(0, ScalarExpr::Literal(_))] = dag.scalar_roots.as_slice() else { - panic!("one literal scalar root: {:?}", dag.scalar_roots) - }; + assert!(check_contract(&empty_plan(), "stub", 0).is_ok()); } /// Registering a name twice fails instead of overwriting, so a comparison diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 5069ca1de..34d3f1f73 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -1,44 +1,31 @@ //! The facade's orchestration of the planning stages (#1047): Stage 0 -//! (the memo of logical groups), Pass 1, Pass 2, Stage 2, Stage 3, and -//! finally the selected plan built through Stage 2's realizer. No stage -//! calls another, and none has a deadline: each runs to completion. +//! (the memo of logical groups), Pass 1, Pass 2, Stage 2 and Stage 3, which +//! builds the selected plan. No stage calls another, and none has a +//! deadline: each runs to completion. + +use asap_logical_asap_optimization::memo_of_queries; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_types::stages::PerComputationLogicalMemoCandidates; -use std::collections::HashSet; use std::rc::Rc; use std::time::Instant; use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; -use asap_logical_asap_optimization::ChosenCandidatePerMemoGroup; -use asap_logical_asap_optimization::DeploymentCapabilitiesError; -use asap_logical_asap_optimization::{ - generate_per_computation_logical_memo_candidates, PerComputationCandidateError, - PerComputationLogicalMemoCandidatesError, -}; -use asap_physical_asap_optimization::{ - generate_physical_asap_memo_candidates, Stage2AssignmentRealizer, -}; -use asap_plan_selection::{select_plan, ErrorSelectingPlan, SearchReport, Selection}; -use asap_types::ir::schema::schema_support::with_promql_series_identity; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{ - AssignmentRealizer, CandidateForMemoGroupId, DagOfSelectedCandidates, - LogicalASAPMemoCandidates, Memo, MemoGroupId, PhysicalASAPMemoCandidates, - PhysicalCandidateError, RealizedPlan, -}; -use asap_types::workload::bounds::with_confirmed_weights_at; +use asap_logical_asap_optimization::{pass1_memo, Pass1Error, PerComputationCandidateError}; +use asap_physical_asap_optimization::generate_physical_asap_memo_candidates; +use asap_plan_selection::{select_plan, ErrorSelectingPlan, Selection}; +use asap_types::deployment_capabilities::DeploymentCapabilitiesError; +use asap_types::ir::schema::series_identity::with_promql_series_identity; +use asap_types::ir::QueryRoot; +use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; +use asap_types::stages::{LogicalASAPMemoCandidates, Memo, PhysicalASAPMemoCandidates}; use asap_types::workload::{DataWorkload, QueryLanguage, QueryWorkloadEntry}; -use crate::pass::{ - OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, PlanningModels, QueryPlan, -}; +use crate::pass::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, PlanningModels}; /// How the facade searches. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct PlanSearchSettings { /// Keep Pass 1's output for inspection (`asap_planner`, the DAG - /// viewer): [`PlanningRun::per_computation_candidates`]. Default off: + /// viewer): [`PlanningRun::pass1_memo`]. Default off: /// production planning copies no memo. pub inspect: bool, } @@ -54,15 +41,9 @@ pub enum SelectionError { /// Pass 1's memo candidates or Pass 2 failed. #[error("logical ASAP-aware optimization: {0}")] LogicalASAPCandidates(String), - /// Stage 2 failed. - #[error("physical ASAP-aware optimization: {0}")] - PhysicalASAPCandidates(String), /// Stage 3 selected no plan. #[error("plan selection: {0}")] Selection(ErrorSelectingPlan), - /// Stage 2 could not build the plan Stage 3 selected. - #[error("building the selected plan: {0}")] - Realization(PhysicalCandidateError), /// The run stopped before plan selection (`asap_planner --run-until`). #[error("plan selection did not run")] NotRun, @@ -76,19 +57,12 @@ pub struct PlanningReport { /// Pass 2's output: memo groups and their candidates. pub memo_groups: usize, pub logical_candidates: usize, - /// Stage 2's output: operator nodes, physical candidates and pruned - /// candidates. - pub operator_nodes: usize, + /// Stage 2's output: physical candidates and pruned candidates. pub physical_candidates: usize, - /// Stage 2's producer links: over every physical candidate and input, - /// the candidates that may produce it. - pub physical_producer_links: usize, pub stage2_pruned_candidates: usize, pub pass2_ms: f64, pub stage2_ms: f64, pub stage3_ms: f64, - /// Stage 3's search report, when it selected a plan. - pub search_report: Option, } /// Every stage's output for one workload ([`run_planning`]). @@ -97,19 +71,14 @@ pub struct PlanningRun { /// Stage 1, Pass 1's output: the memo groups with their per-computation /// candidates and the pruned candidates. /// `None` when Pass 1 failed or without [`PlanSearchSettings::inspect`]. - pub per_computation_candidates: Option, + pub pass1_memo: Option, /// Pass 2's output, when it ran. pub logical_asap_candidates: Option, /// Stage 2's output, when it ran: the catalog `selection`'s candidate /// ids index. pub physical_asap_candidates: Option, - /// Stage 3's selection. - pub selection: Result, - /// The selected plan as Stage 2 built it, with its tumbling-window retention. - pub selected_physical_plan: Option, - /// The plan to run: the selected plan's roots with the weight proofs - /// Stage 3 made applied. - pub selected_plan: Option>, + /// Stage 3's plan and its report of the selection. + pub selection: Result<(PhysicalASAPDAG, Selection), SelectionError>, pub report: PlanningReport, } @@ -127,21 +96,18 @@ pub fn run_planning( for root in &roots { root.validate_structure()?; } - // Stage 0: the memo of logical groups. - let initial = generate_dag_of_logical_memo_groups(&roots); - // Pass 1: the per-computation candidates the deployment can run and - // some query can use. + // Stage 0: the memo of logical groups, each query with its accuracy + // target, or none when accuracy is not enforced. let accuracy_targets: Vec<_> = entries .iter() - .map(|entry| entry.requirements.accuracy.clone()) + .map(|entry| { + (!models.ignore_accuracy_guarantees).then(|| entry.requirements.accuracy.clone()) + }) .collect(); - let accuracy = models.accuracy_check_inputs(); - let pass1 = generate_per_computation_logical_memo_candidates( + let initial = memo_of_queries(&roots, &accuracy_targets, data)?; + // Pass 1: the per-computation candidates the deployment can run. + let pass1 = pass1_memo( initial, - &accuracy_targets, - accuracy.model, - accuracy.evidence, - data, models .deployment_capabilities .supported_summaries @@ -149,24 +115,16 @@ pub fn run_planning( ); // A query Pass 1 cannot plan is an input error. let pass1 = match pass1 { - Err(PerComputationLogicalMemoCandidatesError::PerComputation(error)) => return Err(error), + Err(Pass1Error::PerComputation(error)) => return Err(error), pass1 => pass1, }; let mut report = PlanningReport { pass1_ms: elapsed_ms(started), ..PlanningReport::default() }; - let per_computation_candidates = pass1.as_ref().ok().filter(|_| settings.inspect).cloned(); - let realizer = Stage2AssignmentRealizer { - original_roots: &roots, - entries, - data, - query_time_results_retained: models.deployment_capabilities.query_time_results_retained, - }; + let pass1_memo = pass1.as_ref().ok().filter(|_| settings.inspect).cloned(); let mut logical_asap_candidates = None; let mut physical_asap_candidates = None; - let mut selected_physical_plan = None; - let mut selected_plan = None; let selection = (|| { models .deployment_capabilities @@ -174,76 +132,39 @@ pub fn run_planning( .map_err(SelectionError::InvalidDeploymentCapabilities)?; let pass1 = pass1.map_err(|error| SelectionError::LogicalASAPCandidates(error.to_string()))?; - // Pass 2: local window composition over every legal - // tumbling-window width, then the sharing rules. + // Pass 2: the reuse rules, readout requirements and logical edges. let started = Instant::now(); - let logical = generate_logical_asap_memo_candidates(pass1, entries); + let logical = generate_logical_asap_memo_candidates(pass1, entries, models.evidence); report.pass2_ms = elapsed_ms(started); let logical = logical_asap_candidates.insert( logical.map_err(|error| SelectionError::LogicalASAPCandidates(error.to_string()))?, ); - report.memo_groups = logical.memo.groups.len(); - report.logical_candidates = logical.memo.candidates.iter().map(Vec::len).sum(); + report.memo_groups = logical.memo.groups().len(); + report.logical_candidates = logical + .memo + .groups() + .iter() + .map(|group| group.candidates.len()) + .sum(); // Stage 2. let started = Instant::now(); - let physical = generate_physical_asap_memo_candidates( - logical, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - models.deployment_capabilities.raw_data_retained, - ); + let physical = + generate_physical_asap_memo_candidates(logical, data, models.deployment_capabilities); report.stage2_ms = elapsed_ms(started); - let physical = physical_asap_candidates.insert( - physical.map_err(|error| SelectionError::PhysicalASAPCandidates(error.to_string()))?, - ); - report.operator_nodes = physical.operator_nodes_of_candidates_for_memo_groups.len(); - report.physical_candidates = physical.candidates.len(); - report.physical_producer_links = physical - .candidates - .iter() - .flat_map(|candidate| &candidate.producer_candidates_per_input) - .map(|producers| producers.len()) - .sum(); + let physical = physical_asap_candidates.insert(physical); + report.physical_candidates = physical.physical_candidates.len(); report.stage2_pruned_candidates = physical.pruned_candidates.len(); - // Stage 3. + // Stage 3: the selection and the plan it builds. let started = Instant::now(); - let selection = select_plan( - physical, - entries, - data, - &models.accuracy_check_inputs(), - models.cost_inputs, - &realizer, - ); + let selection = select_plan(physical, entries, data, models.cost_inputs); report.stage3_ms = elapsed_ms(started); - let selection = selection.map_err(SelectionError::Selection)?; - report.search_report = Some(selection.search_report.clone()); - // The selected plan, as Stage 2 builds it. - let realized = realizer - .realize(physical, &selection.selected_dag) - .map_err(SelectionError::Realization)?; - let proven: HashSet<*const OperatorNode> = selection - .nodes_with_proven_non_negative_weights - .iter() - .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) - .map(Rc::as_ptr) - .collect(); - selected_plan = Some(with_confirmed_weights_at( - &realized.plan.roots, - data, - &|node| proven.contains(&Rc::as_ptr(node)), - )); - selected_physical_plan = Some(realized); - Ok(selection) + selection.map_err(SelectionError::Selection) })(); Ok(PlanningRun { - per_computation_candidates, + pass1_memo, logical_asap_candidates, physical_asap_candidates, selection, - selected_physical_plan, - selected_plan, report, }) } @@ -252,89 +173,13 @@ fn elapsed_ms(started: Instant) -> f64 { started.elapsed().as_secs_f64() * 1000.0 } -/// The selected plan's memo candidate for each group it uses: the memo -/// candidate whose root each selected node's candidate is. Factoring may give several -/// memo candidates of a group one root (identical terminal readers of -/// different inputs); the one reading the selected producers' groups is taken. -/// `memo` and `physical` are the run's -/// ([`PlanningRun::logical_asap_candidates`]'s memo and -/// [`PlanningRun::physical_asap_candidates`]); `selected` is the selection's -/// `selected_dag`. An error says why the plan is not one candidate per group. -pub fn selected_chosen_candidates( - memo: &Memo, - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, -) -> Result { - use std::collections::HashMap; - let mut producer_groups = HashMap::<_, Vec>::new(); - for edge in &selected.edges { - producer_groups - .entry(edge.consumer) - .or_default() - .push(physical.memo_group_of(selected.candidate_of_node[edge.producer.0])); - } - let mut chosen_candidates = ChosenCandidatePerMemoGroup::new(); - for (node_id, &id) in selected.candidate_of_node.iter().enumerate() { - let node_id = asap_types::stages::SelectedNodeId(node_id); - let node = &physical.operator_nodes_of_candidates_for_memo_groups - [physical.candidates[id.0].operator_node.0]; - if !node.is_implementation_root { - continue; // a candidate's private node - } - let options = &node.candidates_for_memo_group; - let reads = |&&CandidateForMemoGroupId { group, index }: &&CandidateForMemoGroupId| { - producer_groups.get(&node_id).is_none_or(|read| { - read.iter().all(|&child| { - child == group - || memo.candidates[group.0 as usize][index] - .input_groups - .contains(&child) - }) - }) - }; - let &CandidateForMemoGroupId { group, index } = - options.iter().find(reads).ok_or_else(|| { - format!( - "selected candidate {} reads groups none of its memo candidates read", - id.0 - ) - })?; - if let Some(previous) = chosen_candidates.insert(group, index) { - if previous != index { - return Err(format!( - "the selected plan implements group {} with two candidates, {previous} and {index}", - group.0 - )); - } - } - } - // Every group the chosen candidates need, from the query roots, is chosen. - let mut pending: Vec<_> = asap_types::stages::root_groups(&memo.query_roots) - .into_iter() - .collect(); - let mut seen = HashSet::new(); - while let Some(group) = pending.pop() { - if !seen.insert(group) { - continue; - } - let index = chosen_candidates.get(&group).ok_or_else(|| { - format!( - "the selected plan uses group {} but no candidate of it is selected", - group.0 - ) - })?; - pending.extend(&memo.candidates[group.0 as usize][*index].input_groups); - } - Ok(chosen_candidates) -} - /// The planner pipeline behind the optimization-pass interface: the /// workload's roots in entry order, PromQL roots with each series' full /// identity as a column (the row representation per-series state needs at /// runtime; a root with no such representation is planned over its labels /// alone), through [`run_planning`] with every entry's accuracy target. The -/// output is the selected plan's roots, with the weight proofs plan -/// selection made applied, and its tumbling-window retention. +/// output is the plan Stage 3 built; its `Selection` report is +/// [`run_planning`]'s. #[derive(Debug, Default, Clone, Copy)] pub struct Stage1ToStage3Pass; @@ -375,38 +220,8 @@ impl OptimizationPass for Stage1ToStage3Pass { input.models, PlanSearchSettings::default(), )?; - let selection = run.selection?; - let roots = run.selected_plan.expect("a selection has a plan"); - let mut output = PlanOutput::new(Vec::new()); - for (entry_index, root) in roots.iter().enumerate() { - match root { - QueryRoot::Operator(node) => output.plans.push(QueryPlan { - entry_index, - root: Rc::clone(node), - }), - QueryRoot::Scalar(expr) => { - // One plan per distinct operator the scalar reads. - let mut refs: Vec<&Rc> = Vec::new(); - for node in expr.operator_refs() { - if !refs.iter().any(|r| Rc::ptr_eq(r, node)) { - refs.push(node); - } - } - output - .scalar_plans - .extend(refs.into_iter().map(|node| QueryPlan { - entry_index, - root: Rc::clone(node), - })); - output.scalar_roots.push((entry_index, expr.clone())); - } - } - } - output.tumbling_window_retention_ms = run - .selected_physical_plan - .map(|realized| realized.tumbling_window_retention_ms) - .unwrap_or_default(); - output.selection = Some(selection); + let (plan, _selection) = run.selection?; + let output = PlanOutput::new(plan); Ok(output) } } diff --git a/crates/planner/tests/columnar_batches.rs b/crates/planner/tests/columnar_batches.rs index 658dee91b..adc219edd 100644 --- a/crates/planner/tests/columnar_batches.rs +++ b/crates/planner/tests/columnar_batches.rs @@ -12,9 +12,9 @@ use arrow::array::{DictionaryArray, Float64Array, TimestampMillisecondArray}; use arrow::datatypes::UInt32Type; use asap_executor::physical_planner::compile; use asap_executor::planner::ir::operator::Source; -use asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload as Payload; use asap_executor::planner::ir::schema::PROMQL_SERIES_IDENTITY; use asap_executor::planner::ir::NonASAPOp; +use asap_executor::planner::plan::physical_asap_dag::PhysicalASAPOperatorPayload as Payload; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, BatchColumn}; use asap_planner::{ diff --git a/crates/planner/tests/deployment_accuracy_model.rs b/crates/planner/tests/deployment_accuracy_model.rs deleted file mode 100644 index 91cda2b04..000000000 --- a/crates/planner/tests/deployment_accuracy_model.rs +++ /dev/null @@ -1,374 +0,0 @@ -//! The deployment's accuracy model plugs into plan selection (#509 Stage 3) -//! only. Stage 1 composes guarantees with the built-in analytical rules, so -//! a different model leaves the candidates unchanged and changes only which -//! of them Stage 3 accepts. - -use std::collections::BTreeSet; -use std::rc::Rc; - -use asap_accuracy::{AccuracyModel, BuiltinAccuracyModel}; -use asap_frontend_promql::{lower_promql_query_workload, promql_metric_names}; -use asap_plan_selection::reference_pricer::{select_among_plans, WholePlanSelection}; -use asap_plan_selection::{ErrorSelectingPlan, RejectionCause}; -use asap_planner::PlanningModels; - -/// The reference pricer's selection among every given candidate, and the -/// selected plan's operator roots. -struct PlanOutput { - plans: Vec, - selection: Option, -} - -struct QueryPlan { - root: Rc, -} -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric, AccuracyTarget}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; -use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, -}; - -/// A deployment that trusts no approximate estimate: only an exact -/// guarantee meets a target. -struct ExactOnlyAccuracyModel; - -impl AccuracyModel for ExactOnlyAccuracyModel { - fn satisfies(&self, guarantee: &AccuracyGuarantee, _target: &AccuracyTarget) -> bool { - guarantee.is_exact() - } -} - -/// `query` with a 1% rank error, repeated every ten minutes over metrics -/// sampled every 15 s. -fn rank_target_workload(query: &str) -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(600_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank), - max_latency_ms: None, - }, - predictability: Predictability::Unknown, - range_evaluation: None, - }]), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1.0, - }, - ..DataWorkload::sampled_every(promql_metric_names(query).unwrap(), DurationMs(15_000)) - }), - } -} - -/// Whether `root` reads a sketch state. -fn reads_sketch(root: &Rc) -> bool { - OperatorNode::reachable(root).into_iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) - }) -} - -/// A small, fixed physical inventory, independent of deployment accuracy models. -/// Exercise each local implementation with the remaining computations exact. -/// These fixtures test evaluator hooks, not a timed search's archive prefix. -fn physical_candidates( - workload: &PlanningWorkload, -) -> Vec { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen; - use asap_types::physical::Materialization; - - let roots = lower_promql_query_workload(workload).expect("lowers"); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut choices = BTreeSet::from([vec![0; inventory.targets.len()]]); - for (index, target) in inventory.targets.iter().enumerate() { - for implementation in 1..target.choice_count() { - let mut choice = vec![0; inventory.targets.len()]; - choice[index] = implementation; - choices.insert(choice); - } - } - choices - .into_iter() - .enumerate() - .map(|(index, choice)| { - let roots: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let mut candidate = optimize_physical_asap_chosen( - &format!("fixture-{index}"), - &roots, - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - false, - &mut |_, _, _| Some(Materialization::NotMaterialized), - ) - .unwrap() - .expect("query-time fixture is executable"); - candidate.id = format!("fixture-{index}"); - candidate - }) - .collect() -} - -fn plan_every_candidate( - workload: &PlanningWorkload, - candidates: &[asap_types::physical::PhysicalWorkloadPlan], - models: PlanningModels<'_>, -) -> Result { - let selection = select_among_plans( - candidates, - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - &models.accuracy_check_inputs(), - models.cost_inputs, - )?; - let plans = selection - .plan - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryPlan { - root: Rc::clone(node), - }, - QueryRoot::Scalar(_) => panic!("operator queries only"), - }) - .collect(); - Ok(PlanOutput { - plans, - selection: Some(selection), - }) -} - -/// Every candidate Stage 3 was given: the valid ones it priced and the -/// ones it rejected. -fn stage3_candidate_ids(output: &PlanOutput) -> BTreeSet { - let selection = output.selection.as_ref().expect("plan selection ran"); - selection - .costs - .keys() - .cloned() - .chain(selection.rejected.iter().map(|r| r.id.clone())) - .collect() -} - -/// The candidates for which Stage 3 accepted an approximate output for some -/// query. -fn approximate_accepted(output: &PlanOutput) -> BTreeSet { - let selection = output.selection.as_ref().expect("plan selection ran"); - selection - .accepted - .iter() - .filter(|(_, queries)| queries.iter().any(|q| q.accepted_metric.is_some())) - .map(|(id, _)| id.clone()) - .collect() -} - -#[tokio::test] -async fn the_deployment_accuracy_model_decides_only_in_plan_selection() { - // The user's workload: one p99 query with a 1% rank-error target. - let workload = rank_target_workload("quantile_over_time(0.99, lat[5m])"); - - let candidates = physical_candidates(&workload); - - // The built-in models: Stage 3 checks guarantees with BuiltinAccuracyModel. - let builtin = plan_every_candidate(&workload, &candidates, PlanningModels::builtin()) - .expect("plans with the built-in models"); - // The candidates whose approximate output the built-in model accepts: - // the sketch candidates, sized in Stage 1 for the 1% target. - let builtin_approximate = approximate_accepted(&builtin); - assert!(!builtin_approximate.is_empty()); - - // The same workload with a deployment model that accepts only exact - // outputs, plugged into Stage 3 through `PlanningModels::with_accuracy`. - let exact_only_model = ExactOnlyAccuracyModel; - let exact_only = plan_every_candidate( - &workload, - &candidates, - PlanningModels::builtin().with_accuracy(&exact_only_model), - ) - .expect("plans with the exact-only model"); - - // Stages 1 and 2 do not read the model: Stage 3 got the same - // candidates, with the same guarantees and summary parameters. - assert_eq!( - stage3_candidate_ids(&builtin), - stage3_candidate_ids(&exact_only) - ); - - // Stage 3 now rejects each of those candidates for missing its target, - // accepts no approximate output and selects an exact plan. - let exact_only_selection = exact_only.selection.as_ref().unwrap(); - for id in &builtin_approximate { - assert!(exact_only_selection - .rejected - .iter() - .any(|r| &r.id == id && r.cause == RejectionCause::AccuracyTargetMissed)); - } - assert!(approximate_accepted(&exact_only).is_empty()); - assert!(!reads_sketch(&exact_only.plans[0].root)); -} - -/// A deployment that has measured every output itself and accepts each one -/// for any target, whatever its guarantee says. -struct AcceptsEveryOutput; - -impl AccuracyModel for AcceptsEveryOutput { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - BuiltinAccuracyModel.satisfies(guarantee, target) - } - - fn accepts_output_regardless_of_guarantee( - &self, - _output: &QueryRoot, - _target: &AccuracyTarget, - ) -> bool { - true - } -} - -/// The candidates for which Stage 3 accepted some query's output because -/// the deployment's model accepted it regardless of its guarantee. -fn accepted_regardless_of_guarantee(output: &PlanOutput) -> BTreeSet { - let selection = output.selection.as_ref().expect("plan selection ran"); - selection - .accepted - .iter() - .filter(|(_, queries)| queries.iter().any(|q| q.accepted_regardless_of_guarantee)) - .map(|(id, _)| id.clone()) - .collect() -} - -/// The ids of the candidates `output`'s selection rejected with `cause`. -fn rejected_with(output: &PlanOutput, cause: RejectionCause) -> BTreeSet { - output - .selection - .as_ref() - .expect("plan selection ran") - .rejected - .iter() - .filter(|r| r.cause == cause) - .map(|r| r.id.clone()) - .collect() -} - -/// A deployment model that accepts an output directly also accepts one with -/// no guarantee: the built-in model rejects the summary candidates of a -/// difference of two quantile estimates as having none, and the accepting -/// model makes them valid, and Stage 3 selects one. -#[tokio::test] -async fn the_deployment_accuracy_model_accepts_outputs_without_a_guarantee() { - // The user's workload: a difference of two quantile estimates, 1% rank - // error. Stage 1 has no analytical rule for a difference of rank - // estimates, so it keeps the summary candidates with no guarantee. - let workload = rank_target_workload("quantile(0.99, rate(x[5m])) - quantile(0.5, rate(x[5m]))"); - - let candidates = physical_candidates(&workload); - - // The built-in model (default hook: accepts nothing directly) rejects - // every summary candidate as having no guarantee and selects exact. - let builtin = plan_every_candidate(&workload, &candidates, PlanningModels::builtin()) - .expect("plans with the built-in models"); - // Candidates rejected for having no guarantee. - let unguaranteed = rejected_with(&builtin, RejectionCause::NoAccuracyGuarantee); - assert!(!unguaranteed.is_empty()); - assert!(accepted_regardless_of_guarantee(&builtin).is_empty()); - assert!(!reads_sketch(&builtin.plans[0].root)); - - // The deployment's model accepts every output directly. - let accepting_model = AcceptsEveryOutput; - let accepting = plan_every_candidate( - &workload, - &candidates, - PlanningModels::builtin().with_accuracy(&accepting_model), - ) - .expect("plans with the accepting model"); - let accepting_selection = accepting.selection.as_ref().unwrap(); - assert_eq!( - stage3_candidate_ids(&builtin), - stage3_candidate_ids(&accepting) - ); - - // Every candidate the built-in model rejected for having no guarantee - // is now valid and priced, accepted regardless of its guarantee, and no - // candidate fails the accuracy check. - assert!(accepted_regardless_of_guarantee(&accepting).is_superset(&unguaranteed)); - for id in &unguaranteed { - assert!(accepting_selection.costs.contains_key(id), "{id}"); - } - assert!(accepting_selection.rejected.iter().all(|r| !matches!( - r.cause, - RejectionCause::NoAccuracyGuarantee | RejectionCause::AccuracyTargetMissed - ))); - // A singleton previously rejected sketch is now selectable; the cheapest - // plan in the complete fixture need not be that sketch. - let sketch = candidates - .iter() - .find(|c| unguaranteed.contains(&c.id)) - .unwrap(); - let singleton = plan_every_candidate( - &workload, - std::slice::from_ref(sketch), - PlanningModels::builtin().with_accuracy(&accepting_model), - ) - .unwrap(); - assert_eq!(singleton.selection.as_ref().unwrap().selected, sketch.id); - assert!(reads_sketch(&singleton.plans[0].root)); -} - -/// The hook has priority over a guarantee that misses: a candidate whose -/// guarantee misses its target under the built-in models (a relative bound -/// for a rank target) is accepted under the accepting model. -#[tokio::test] -async fn the_deployment_accuracy_model_accepts_outputs_whose_guarantee_misses() { - // The user's workload: one p99 query with a 1% rank-error target. - let workload = rank_target_workload("quantile_over_time(0.99, lat[5m])"); - let candidates = physical_candidates(&workload); - // Under the built-in models: the candidates whose guarantee misses the - // target (a relative bound for a rank target). - let builtin = plan_every_candidate(&workload, &candidates, PlanningModels::builtin()) - .expect("plans with the built-in models"); - let missed = rejected_with(&builtin, RejectionCause::AccuracyTargetMissed); - assert!(!missed.is_empty()); - // The same workload with the model that accepts every output directly. - let accepting_model = AcceptsEveryOutput; - let accepting = plan_every_candidate( - &workload, - &candidates, - PlanningModels::builtin().with_accuracy(&accepting_model), - ) - .expect("plans with the accepting model"); - // Each of them is now valid, priced and accepted regardless of guarantee. - let accepting_selection = accepting.selection.as_ref().unwrap(); - for id in &missed { - assert!(accepting_selection.costs.contains_key(id), "{id}"); - assert!( - accepting_selection.accepted[id][0].accepted_regardless_of_guarantee, - "{id}" - ); - } - // The hook also overrides guarantees that meet the target: every - // accepted output, including exact and KLL candidates, is accepted - // regardless of its guarantee. - assert_eq!(accepting_selection.accepted.len(), candidates.len()); - for (id, queries) in &accepting_selection.accepted { - assert!( - queries.iter().all(|q| q.accepted_regardless_of_guarantee), - "{id}" - ); - } -} diff --git a/crates/planner/tests/e2e_plan.rs b/crates/planner/tests/e2e_plan.rs index f32b355b4..4702aba5e 100644 --- a/crates/planner/tests/e2e_plan.rs +++ b/crates/planner/tests/e2e_plan.rs @@ -1,8 +1,6 @@ //! End-to-end coverage for the library facade (#429) and for plugging a //! different optimization pass into it (#430). -use std::rc::Rc; - use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; use asap_planner::PlanningModels; use asap_planner::{e2e_plan, FrontendInput, PlanError, UserInput, UserInputError}; @@ -10,7 +8,7 @@ use asap_planner::{ run_planning, OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, Stage1ToStage3Pass, }; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, @@ -102,12 +100,12 @@ async fn plans_every_query_in_entry_order() { ); let output = e2e_plan(input).await.expect("workload plans"); - assert_eq!(output.plans.len(), 2); - assert_eq!(output.entry_indices(), vec![0, 1]); + assert_eq!(output.plan.roots.len(), 2); + assert!(output.plan.scalar_roots.is_empty()); } -/// The facade selects what the planning stages select over the frontends' -/// roots ([`run_planning`]), and reports that selection. +/// The facade returns the plan the planning stages build over the frontends' +/// roots ([`run_planning`]). #[tokio::test] async fn facade_plans_match_run_planning() { let workload = sql_workload( @@ -134,14 +132,9 @@ async fn facade_plans_match_run_planning() { let mut targets = Vec::new(); for (index, entry) in workload.query_workload.entries().enumerate() { let accuracy = entry.requirements.accuracy.clone(); - let expr = lower_sql_dialect( - &entry.query.0, - &catalog, - SqlDialect::DataFusionSQL, - accuracy.clone(), - ) - .await - .expect("lowers"); + let expr = lower_sql_dialect(&entry.query.0, &catalog, SqlDialect::DataFusionSQL) + .await + .expect("lowers"); roots.push((index, expr.into())); targets.push(Some(accuracy)); } @@ -163,16 +156,8 @@ async fn facade_plans_match_run_planning() { asap_planner::PlanSearchSettings::default(), ) .unwrap(); - let selection = run.selection.as_ref().expect("a candidate is selected"); - // Separate searches have different timings. The facade must preserve - // the chosen execution, cost, and guarantees. - let facade = output.selection.as_ref().expect("facade reports selection"); - assert_eq!(facade.cost, selection.cost); - assert_eq!( - facade.accepted_accuracy_per_query, - selection.accepted_accuracy_per_query - ); - assert_eq!(&output.roots(), run.selected_plan.as_ref().unwrap()); + let (plan, _selection) = run.selection.as_ref().expect("a candidate is selected"); + assert_eq!(&output.plan, plan); } /// A repeating SQL query reaches the optimizer, not only `query_batch` entries. @@ -199,11 +184,10 @@ async fn lowers_repeating_sql_entries_too() { let output = e2e_plan(input).await.expect("workload plans"); assert_eq!( - output.len(), + output.plan.roots.len(), 2, "batch entry and repeating entry both planned" ); - assert_eq!(output.entry_indices(), vec![0, 1]); } /// A caller-supplied pass replaces the shipped algorithm entirely: the trait is @@ -247,10 +231,10 @@ async fn runs_a_caller_supplied_pass_instead_of_the_shipped_one() { assert_eq!(detail, "saw 1 entry/entries"); } -/// The harness catches a pass that mislabels which query a plan belongs to, -/// rather than letting the mislabelled plan reach a deployment. +/// The harness catches a pass whose plan answers fewer queries than the +/// workload has, rather than letting the plan reach a deployment. #[tokio::test] -async fn harness_rejects_a_pass_that_mislabels_entry_indices() { +async fn harness_rejects_a_pass_that_drops_a_query() { struct Mangling; impl OptimizationPass for Mangling { @@ -259,9 +243,7 @@ async fn harness_rejects_a_pass_that_mislabels_entry_indices() { } fn optimize(&self, input: OptimizationInput<'_>) -> Result { let mut output = Stage1ToStage3Pass.optimize(input)?; - for plan in output.plans.iter_mut() { - plan.entry_index += 1; - } + output.plan.roots.pop(); Ok(output) } } @@ -309,7 +291,7 @@ async fn rejects_a_frontend_that_does_not_match_the_workload_language() { )); } -/// A repeating PromQL query yields one plan carrying its selected DAG root. +/// A repeating PromQL query yields a plan with one root. #[tokio::test] async fn each_plan_carries_its_selected_root() { let workload = PlanningWorkload { @@ -329,6 +311,7 @@ async fn each_plan_carries_its_selected_root() { data_workload: Some(DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1_000.0, + max_out_of_order_ms: None, }, ..Default::default() }), @@ -336,10 +319,7 @@ async fn each_plan_carries_its_selected_root() { let input = UserInput::new(&workload, FrontendInput::Promql, PlanningModels::builtin()); let output = e2e_plan(input).await.expect("workload plans"); - assert_eq!(output.plans.len(), 1); - assert_eq!(output.plans[0].entry_index, 0); - let _: &Rc<_> = &output.plans[0].root; - assert_eq!(output.operator_roots().len(), 1); + assert_eq!(output.plan.roots.len(), 1); } /// Scalar-only and mixed workloads preserve entry bindings without wrapper nodes. @@ -372,25 +352,15 @@ async fn scalar_roots_survive_planning_in_workload_order() { )) .await .unwrap(); - assert_eq!( - output.entry_indices(), - (0..queries.len()).collect::>() - ); - assert!(matches!( - output.roots()[0], - asap_types::ir::QueryRoot::Scalar(_) - )); - assert_eq!(output.roots().len(), queries.len()); + let roots = output.plan.query_roots().unwrap(); + assert!(matches!(roots[0], asap_types::ir::QueryRoot::Scalar(_))); + assert_eq!(roots.len(), queries.len()); if queries.len() == 3 { - assert_eq!(output.plans[0].entry_index, 1); - let asap_types::ir::QueryRoot::Scalar(expr) = &output.roots()[2] else { + assert!(matches!(roots[1], asap_types::ir::QueryRoot::Operator(_))); + let asap_types::ir::QueryRoot::Scalar(expr) = &roots[2] else { panic!() }; assert_eq!(expr.operator_refs().len(), 1); - assert!(Rc::ptr_eq( - expr.operator_refs()[0], - &output.scalar_plans[0].root - )); } } } diff --git a/crates/planner/tests/main.rs b/crates/planner/tests/main.rs index bb91f08fe..9d06c8c46 100644 --- a/crates/planner/tests/main.rs +++ b/crates/planner/tests/main.rs @@ -5,10 +5,7 @@ mod time_series; mod columnar_batches; -mod deployment_accuracy_model; mod e2e_plan; -mod per_sample_updater; mod runtime_install; -mod sql_per_group_topk; mod sql_table_scans; mod summary_sharing; diff --git a/crates/planner/tests/per_sample_updater.rs b/crates/planner/tests/per_sample_updater.rs deleted file mode 100644 index b8f83f1e5..000000000 --- a/crates/planner/tests/per_sample_updater.rs +++ /dev/null @@ -1,405 +0,0 @@ -//! A runtime that ingests sample by sample refreshes a materialized tumbling window's states -//! with `StateColumnUpdater`, the code the plan's `SummaryAgg` runs on a -//! batch (issue #762): the same states as the executor's tumbling-window build. - -use std::collections::BTreeMap; -use std::sync::Arc; - -use asap_executor::physical_planner::compile; -use asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload as Payload; -use asap_executor::planner::ir::ASAPOp; -use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::summary_kernels::exact::ExactAccumulator; -use asap_executor::StateColumnUpdater; -use asap_planner::{ - plan_installable, test_support::query_time_reference, DeploymentCapabilities, FrontendInput, - PlanningModels, PlanningWorkload, UserInput, -}; -use futures::StreamExt; - -use crate::time_series; -use time_series::{series_identity, series_labels, time_series_tables, RawDataSample}; - -/// The evaluation time: the end of an hour. -const EVALUATION_MS: i64 = 3_600_000; - -#[tokio::test] -async fn per_sample_updates_build_the_materialized_tumbling_window_the_plan_refreshes() { - // What the user writes: the 5-minute rate of a counter, every minute, - // exactly, over 1,000 series sampled every 15 s. - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "rate(http_requests_total[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 66.7}}, - "sources": {"http_requests_total": {"series_count": 1000, "sample_interval": 15000}}}}"#, - ) - .unwrap(); - // No raw data samples retained: the plan stores rate tumbling windows built at ingestion. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - // The installable plan: the selected DAG and its materialized views. - let plan = plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .unwrap() - .0; - let dag = &plan.dag; - // The newest tumbling window view and its window at this evaluation. - let views = plan.materialized_views().unwrap(); - let output = views - .iter() - .max_by_key(|output| { - output.inputs.raw_data_windows()[0] - .bounds_ms(EVALUATION_MS) - .1 - }) - .unwrap(); - let (start_ms, end_ms) = output.inputs.raw_data_windows()[0].bounds_ms(EVALUATION_MS); - // Five one-minute tumbling windows, the newest (59 min, 60 min]. - assert_eq!(views.len(), 5); - assert_eq!((start_ms, end_ms), (3_540_000, 3_600_000)); - // The view node's summary family, as the plan declares it: the - // `SummaryAgg` at or below the view node. - let family = { - let mut id = output.node_id; - loop { - let node = dag.nodes.iter().find(|n| n.id == id).unwrap(); - if let Payload::ASAP(ASAPOp::SummaryAgg { family, .. }) = &node.payload { - break family.clone(); - } - id = dag - .edges - .iter() - .find(|e| e.consumer == id) - .unwrap() - .producer; - } - }; - // What the runtime received in the tumbling window: 3 series, one counter sample - // every 15 s, one series with a reset. - assert_eq!( - family, - asap_executor::planner::ir::schema::FieldDataType::ExactAggregate( - asap_executor::planner::ir::schema::ExactKind::Rate, - asap_executor::planner::ir::schema::ExactParams::Rate, - ) - ); - let samples: Vec = ["a", "b", "c"] - .iter() - .flat_map(|instance| { - let mut value = 0.0; - (1..=(end_ms - start_ms) / 15_000).map(move |step| { - value = if *instance == "c" && step == 3 { - 2.0 - } else { - value + 10.0 - }; - RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "http_requests_total".to_string()), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: start_ms + step * 15_000, - value, - } - }) - }) - .collect(); - - // The executor's tumbling-window build: the samples bound at the plan's `Scan`, - // its `TimeRange` keeping the time range the tumbling window covers. - let compiled = compile(dag, &[], &[output.node_id as u64]).unwrap(); - let build = compiled - .instantiate(Arc::new(time_series_tables(dag, &samples))) - .unwrap(); - let context = RunContext::new( - Scope::Ingestion { - window_start_ms: start_ms, - window_end_ms: end_ms, - }, - Limits::default(), - ) - .unwrap(); - let tumbling_window = build - .execute(compiled.roots(), context) - .unwrap() - .remove(0) - .next() - .await - .unwrap() - .unwrap(); - // Its states, by series labels. - let state_column = tumbling_window - .schema() - .fields - .iter() - .position(|field| !field.is_plain()) - .unwrap(); - let from_dag: BTreeMap, String> = tumbling_window - .to_rows() - .iter() - .map(|row| { - let asap_executor::values::Value::Summary { state, .. } = &row[state_column] else { - panic!("a state") - }; - let exact = state.as_any().downcast_ref::().unwrap(); - ( - series_labels(tumbling_window.schema(), row), - serde_json::to_string(exact).unwrap(), - ) - }) - .collect(); - - // The per-sample build: one updater for the tumbling window, a group per series. - let mut updater = StateColumnUpdater::new(&family).unwrap(); - let mut groups: BTreeMap, u32> = BTreeMap::new(); - for sample in &samples { - let group = match groups.get(&sample.labels) { - Some(&group) => group, - None => { - let group = updater.add_group().unwrap(); - groups.insert(sample.labels.clone(), group); - group - } - }; - updater - .update(group, sample.value, sample.timestamp_ms) - .unwrap(); - } - // Sealed at tumbling window close: one state per group. - let states = updater.seal(); - let per_sample: BTreeMap, String> = groups - .into_iter() - .map(|(mut labels, group)| { - let state = states.state(group as usize); - let exact = state.as_any().downcast_ref::().unwrap(); - // A rate's series drops its metric name, as the DAG's does. - if !from_dag.keys().any(|l| l.contains_key("__name__")) { - labels.remove("__name__"); - } - (labels, serde_json::to_string(exact).unwrap()) - }) - .collect(); - assert_eq!(from_dag.len(), 3); - assert_eq!(per_sample, from_dag); -} - -/// Run `ids` of `dag` with the view states `inputs` bound at their nodes -/// and the raw `samples` at its `Scan`s, in `scope`: each id's answer rows -/// as (series labels, value), sorted. -async fn answers( - dag: &asap_planner::PhysicalASAPDAG, - inputs: BTreeMap, - samples: &[RawDataSample], - ids: &[u64], - scope: Scope, -) -> Vec, f64)>> { - // The plan's views whose states are bound; each one's schema is its DAG - // node's. - let views: Vec<_> = dag - .materialized_views() - .unwrap() - .into_iter() - .filter(|view| inputs.contains_key(&(view.node_id as u64))) - .collect(); - assert_eq!(views.len(), inputs.len()); - // The executor's plan for `ids`, reading the bound views. - let compiled = compile(dag, &views, ids).unwrap(); - // Each input as a view's states in an in-memory store. - let mut store = time_series_tables(dag, samples); - for (id, batch) in inputs { - store = store.with_materialized_view(id as _, vec![batch]).unwrap(); - } - let instance = compiled.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new(scope, Limits::default()).unwrap(); - let mut results = Vec::new(); - for mut output in instance.execute(compiled.roots(), context).unwrap() { - let mut rows = Vec::new(); - while let Some(batch) = output.next().await { - let batch = batch.unwrap(); - for row in &batch.to_rows() { - let labels = series_labels(batch.schema(), row); - let value = row - .iter() - .find_map(|v| match v { - asap_executor::values::Value::Float64(v) => Some(*v), - _ => None, - }) - .unwrap(); - rows.push((labels, value)); - } - } - rows.sort_by(|a, b| a.0.cmp(&b.0)); - results.push(rows); - } - results -} - -/// A runtime builds every materialized tumbling window sample by sample, installs each as a -/// batch with `Batch::try_from_columns` (no DAG run), and answers the query -/// from them: the same answers as the plan's query-time reference over the -/// raw data samples. -#[tokio::test] -async fn tumbling_windows_installed_from_per_sample_states_answer_the_query() { - use asap_executor::arrow::array::{DictionaryArray, TimestampMillisecondArray, UInt32Array}; - use asap_executor::arrow::datatypes::UInt32Type; - use asap_executor::values::{Batch, BatchColumn}; - use std::sync::Arc; - // What the user writes: the 5-minute rate of a counter, every minute, - // exactly, over 1,000 series sampled every 15 s. - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "rate(http_requests_total[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 66.7}}, - "sources": {"http_requests_total": {"series_count": 1000, "sample_interval": 15000}}}}"#, - ) - .unwrap(); - // No raw data samples retained: five one-minute rate tumbling windows built at ingestion. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan = plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .unwrap() - .0; - let dag = &plan.dag; - // What the runtime received: 3 series, a counter sample every 15 s over - // the hour (one series resets), the last at the evaluation time. - let samples: Vec = ["a", "b", "c"] - .iter() - .flat_map(|instance| { - let mut value = 0.0; - (1..=EVALUATION_MS / 15_000).map(move |step| { - value = if *instance == "c" && step == 230 { - 3.0 - } else { - value + 7.0 - }; - RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "http_requests_total".to_string()), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: step * 15_000, - value, - } - }) - }) - .collect(); - // Every materialized tumbling window, built sample by sample and installed as a batch. - let mut tumbling_windows = BTreeMap::new(); - for output in &plan.materialized_views().unwrap() { - // The time range the tumbling window covers, `(start_ms, end_ms]`, at this evaluation. - let (start_ms, end_ms) = output.inputs.raw_data_windows()[0].bounds_ms(EVALUATION_MS); - // The view node's family and output schema, from the plan. - let node = dag.nodes.iter().find(|n| n.id == output.node_id).unwrap(); - let mut id = output.node_id; - let family = loop { - let node = dag.nodes.iter().find(|n| n.id == id).unwrap(); - if let Payload::ASAP(ASAPOp::SummaryAgg { family, .. }) = &node.payload { - break family.clone(); - } - id = dag - .edges - .iter() - .find(|e| e.consumer == id) - .unwrap() - .producer; - }; - // The runtime's per-sample build: a group per series ... - let mut updater = StateColumnUpdater::new(&family).unwrap(); - let mut identities: Vec = Vec::new(); - let mut group_of_series: BTreeMap, u32> = BTreeMap::new(); - for sample in samples - .iter() - .filter(|s| start_ms < s.timestamp_ms && s.timestamp_ms <= end_ms) - { - let group = match group_of_series.get(&sample.labels) { - Some(&group) => group, - None => { - let group = updater.add_group().unwrap(); - group_of_series.insert(sample.labels.clone(), group); - identities.push(series_identity(&sample.labels)); - group - } - }; - updater - .update(group, sample.value, sample.timestamp_ms) - .unwrap(); - } - // ... sealed at tumbling window close: one rate record per series. - let states = Arc::new(updater.seal()); - // The tumbling window's columns, in the view node's schema: the tumbling window end as - // its time, each series' state, and its identity. - let schema = Arc::new(node.output_schema.clone()); - let groups = identities.len(); - let columns = schema - .fields - .iter() - .enumerate() - .map(|(index, field)| { - if Some(index) == schema.time_index { - BatchColumn::Plain(Arc::new(TimestampMillisecondArray::from(vec![ - end_ms; - groups - ]))) - } else if !field.is_plain() { - BatchColumn::States { - indices: UInt32Array::from_iter_values(0..groups as u32), - states: states.clone(), - } - } else { - BatchColumn::Plain(Arc::new( - identities - .iter() - .map(String::as_str) - .collect::>(), - )) - } - }) - .collect(); - // Installed without running the DAG. - tumbling_windows.insert( - output.node_id as u64, - Batch::try_from_columns(schema, columns).unwrap(), - ); - } - assert_eq!(tumbling_windows.len(), 5); - let query = Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }; - let roots: Vec = plan.dag.roots.iter().map(|&r| r as u64).collect(); - // The query's answers from the installed tumbling windows. - let from_tumbling_windows = - answers(dag, tumbling_windows, &samples, &roots, query.clone()).await; - // The plan's query-time reference over the raw data samples. - let reference = query_time_reference(&plan).unwrap(); - let reference_roots: Vec = reference.dag.roots.iter().map(|&r| r as u64).collect(); - let expected = answers( - &reference.dag, - BTreeMap::new(), - &samples, - &reference_roots, - query, - ) - .await; - // One rate per series, equal to the reference. - assert_eq!(from_tumbling_windows[0].len(), 3); - assert_eq!(from_tumbling_windows, expected); -} diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index 7b952acb2..f345ee885 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -16,28 +16,15 @@ use asap_executor::sources::{ }; use asap_executor::values::{Batch, Value}; use asap_executor::Error as ExecutorError; -use asap_physical_asap_optimization::test_support::split_shared_by_timing; use asap_planner::{ - plan_installable, run_planning, CostInputs, DeploymentCapabilities, FrontendInput, - InstallablePlan, InstallablePlanError, MaterializedView, MaterializedViewRefresh, OperatorKind, - OptimizationInput, OptimizationPass, OptimizeError, PhysicalASAPDAG, PlanOutput, - PlanSearchSettings, PlanningModels, PlanningWorkload, RefreshWindow, SelectedPlanReport, - SelectionBox, Stage1ToStage3Pass, UserInput, + plan_installable, CostInputs, DeploymentCapabilities, FrontendInput, InstallablePlan, + MaterializedView, OperatorKind, PhysicalASAPDAG, PlanningModels, PlanningWorkload, + SelectedPlanReport, UserInput, }; use asap_types::ir::operator::Source; -use asap_types::ir::schema::schema_support::with_promql_series_identity; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, OperatorNode, - QueryRoot, -}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::stages::HowToComputeAggregateMeasure; -use asap_types::stages::PruneReason; + use futures::{executor::block_on, StreamExt}; -use std::collections::HashMap; -use std::ops::Bound; -use std::rc::Rc; + use time_series::{series_labels, time_series_tables, RawDataSample}; /// Example 4b's evaluation time: the end of the 60-minute window. @@ -80,31 +67,6 @@ fn sample(metric: &str, instance: &str, timestamp_ms: i64, value: f64) -> RawDat } } -/// Raw `latency_ms` samples of series `a` and `b`, one every 10 s from 70 -/// minutes before `EVALUATION_MS` to a minute after it (for a second -/// evaluation), off the tumbling-window boundaries, and one -/// sample of another metric the plan never reads. A sample's value says -/// which 10-minute tumbling window before `EVALUATION_MS` it falls in: `1000 × tumbling_window + -/// base + 0..=100`, with base 100 for `a` and 300 for `b` (tumbling window 0 is the -/// newest), so an answer read from the wrong tumbling windows is visibly off. -fn remote_write_samples() -> Vec { - let mut samples = Vec::new(); - for (instance, base) in [("a", 100.0), ("b", 300.0)] { - for step in 0..426 { - let timestamp_ms = EVALUATION_MS - 4_195_000 + step * 10_000; - let tumbling_window = (EVALUATION_MS - timestamp_ms) / 600_000; - samples.push(sample( - "latency_ms", - instance, - timestamp_ms, - (1000 * tumbling_window) as f64 + base + (step * 37 % 101) as f64, - )); - } - } - samples.push(sample("cpu_seconds", "a", 5_000, 1e9)); - samples -} - thread_local! { /// How many `read_raw_data` calls the runs on this thread made. static RAW_DATA_READS: Cell = const { Cell::new(0) }; @@ -206,46 +168,9 @@ fn answers(batches: &[Batch]) -> Vec { answers } -/// Each answer is a p99 read from tumbling window `tumbling_window` of its series -/// ([`remote_write_samples`]): within `1000 × tumbling_window + base + 0..=100`. -fn assert_answers_from_tumbling_window(answers: &[Answer], tumbling_window: i64) { - for (labels, value) in answers { - let base = if labels["instance"] == "a" { - 100.0 - } else { - 300.0 - }; - let low = (1000 * tumbling_window) as f64 + base; - assert!( - (low..=low + 100.0).contains(value), - "{labels:?}: {value} is not from tumbling window {tumbling_window}" - ); - } -} - -/// The control plane: plan `queries` for a deployment that keeps no raw -/// samples, with its cost model read from JSON, and write the plan file. -async fn plan_json(queries: &[&str]) -> String { - // What the deployment keeps: no raw data samples, no query-time results - // across evaluations. Supported summaries are left unrestricted, so the - // Planner offers only what its own executor runs. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - plan_json_for(workload(queries), &capabilities).await -} - -/// [`plan_json`] for `workload` on a deployment with `capabilities`. -async fn plan_json_for( - workload: PlanningWorkload, - capabilities: &DeploymentCapabilities, -) -> String { - serde_json::to_string(&plan_with_report_for(workload, capabilities).await.0).unwrap() -} - -/// The plan [`plan_json`] writes for `queries`, and Stage 3's report of the -/// selection it made (which the plan file does not carry). +/// The plan for `queries` on a deployment that keeps no raw samples, and +/// Stage 3's report of the selection it made (which the plan file does not +/// carry). async fn planned_with_report(queries: &[&str]) -> (InstallablePlan, Option) { let capabilities = DeploymentCapabilities { raw_data_retained: false, @@ -280,453 +205,11 @@ async fn plan_with_report_for( .unwrap() } -/// What one data-plane evaluation of a plan file returned. -struct DataPlaneRun { - /// Per query, in workload order, the installed plan's answers. - answers_per_query: Vec>, - /// Per query, the query-time reference's answers. - reference_answers_per_query: Vec>, - /// How many materialized views this evaluation refreshed (ingestion or - /// query runs); the others it read came from the store. - outputs_built: usize, - /// How many `read_raw_data` calls those refreshes made. - raw_data_reads_refreshing: usize, -} - -/// One summary state this test stores: its refreshed state, and how long -/// to keep it. -struct StoredState { - /// The refreshed state, bound at every materialized view with its key. - state: Batch, - /// When the data it summarizes starts, in Unix milliseconds. - start_ms: i64, - /// How far back from the newest evaluation it is kept: the largest - /// `retention_ms` among the views it served. - retention_ms: u64, -} - -/// This stand-in runtime's summary store, by [`state_key`]. -type SummaryStore = BTreeMap; - -/// The key of `view`'s state at evaluation `evaluation_ms`, as a summary -/// store keys instances: its definition id and its selection made -/// concrete (relative time made absolute). A view whose selection -/// names no time (its definition reads a window relative to the -/// evaluation, e.g. an instant selector) is keyed by the evaluation too. -fn state_key(view: &MaterializedView, evaluation_ms: i64) -> String { - use std::ops::Bound; - let absolute = |bound: &Bound| match bound { - Bound::Included(at) => Bound::Included(evaluation_ms + at), - Bound::Excluded(at) => Bound::Excluded(evaluation_ms + at), - Bound::Unbounded => Bound::Unbounded, - }; - let boxes = view.coverage().selection(); - let timed = boxes.iter().any(|b| b.relative_time().is_some()); - let selection: Vec = boxes - .iter() - .map(|selection_box| { - SelectionBox::new( - selection_box.columns().clone(), - selection_box - .relative_time() - .map(|(lower, upper)| (absolute(&lower), absolute(&upper))), - ) - }) - .collect(); - serde_json::json!({ - "definition_id": view.coverage().definition_id(), - "selection": selection, - "evaluation_ms": (!timed).then_some(evaluation_ms), - }) - .to_string() -} - /// The node that answers each query of `plan`, in workload order. fn query_roots(plan: &InstallablePlan) -> Vec { plan.dag.roots.iter().map(|&root| root as u64).collect() } -/// The data plane at evaluation time `evaluation_ms`: from `plan_json`, -/// `samples` and the states earlier evaluations left in `store`, refresh each -/// materialized view the store lacks once, from its raw-data windows and the -/// views it reads: the views refreshed at ingestion by refresh group, one -/// ingestion run per group and span, then the others; answer every query; drop the -/// states past their retention; and answer the queries again with the -/// query-time reference. -fn run_data_plane( - plan_json: &str, - samples: &[RawDataSample], - evaluation_ms: i64, - store: &mut SummaryStore, -) -> DataPlaneRun { - run_data_plane_on( - plan_json, - samples, - evaluation_ms, - store, - IngestionBuilds::PerEvaluation, - ) -} - -/// When this stand-in runtime refreshes a materialized view an ingestion -/// run refreshes from raw data samples. -#[derive(Clone, Copy)] -enum IngestionBuilds { - /// For each evaluation, over windows ending at that evaluation. - PerEvaluation, - /// Tumbling, as a runtime that maintains tumbling windows as samples arrive: each - /// once, over windows ending on the grid of its window's width anchored - /// at the epoch (tumbling window end ≡ 0 mod width, as the backend data plane - /// builds them); an evaluation reads the newest one at or before it. - Tumbling, -} - -/// [`run_data_plane`] with at-ingestion views refreshed as `ingestion` says. -fn run_data_plane_on( - plan_json: &str, - samples: &[RawDataSample], - evaluation_ms: i64, - store: &mut SummaryStore, - ingestion: IngestionBuilds, -) -> DataPlaneRun { - // The installed plan, read back from its file and checked. - let installed: InstallablePlan = serde_json::from_str(plan_json).unwrap(); - installed.validate().unwrap(); - // The DAG the executor runs. - let dag = &installed.dag; - let views = installed.materialized_views().unwrap(); - // Every input of the query run, keyed by the node it is bound to. - let mut inputs = BTreeMap::new(); - // How many materialized views this evaluation refreshes. - let mut outputs_built = 0; - // Each view's state key and the time span of its data at this - // evaluation, `(start_ms, end_ms]`, by node id. - let mut planned: BTreeMap = BTreeMap::new(); - for view in &views { - // The evaluation its windows end at: this one, or for a tumbling - // at-ingestion view the newest point of its grid at or before it. - let built_for_ms = match (ingestion, view.refresh()) { - (IngestionBuilds::Tumbling, MaterializedViewRefresh::AtIngestion) => view - .inputs - .raw_data_windows() - .first() - .map_or(evaluation_ms, |window| { - evaluation_ms - evaluation_ms.rem_euclid(window.width_ms()) - }), - _ => evaluation_ms, - }; - // The span of the data it summarizes: its windows' and the views' - // it reads spans together. - let span = view - .inputs - .raw_data_windows() - .iter() - .map(|window| window.bounds_ms(built_for_ms)) - .chain( - view.inputs - .materialized_views - .iter() - .map(|id| planned[&(*id as u64)].1), - ) - .reduce(|(a, b), (c, d)| (a.min(c), b.max(d))) - .expect("a materialized view reads something"); - planned.insert(view.node_id as u64, (state_key(view, built_for_ms), span)); - } - // The runs that refresh the views the store lacks: for each refresh - // group, one ingestion run per span its views' data covers, with every - // view of the group over that span as a root (a node they share runs - // once); then each view refreshed on first read in this evaluation's - // query run. - let mut refreshes: Vec<(Vec<&MaterializedView>, Scope)> = Vec::new(); - for group in installed.ingestion_refresh_groups().unwrap() { - let mut by_span: BTreeMap<(i64, i64), Vec<&MaterializedView>> = BTreeMap::new(); - for member in &group.views { - let view = views - .iter() - .find(|view| view.node_id == member.node_id) - .unwrap(); - by_span - .entry(planned[&(view.node_id as u64)].1) - .or_default() - .push(view); - } - for (span, members) in by_span { - let scope = Scope::Ingestion { - window_start_ms: span.0, - window_end_ms: span.1, - }; - refreshes.push((members, scope)); - } - } - for view in views - .iter() - .filter(|view| view.refresh() == MaterializedViewRefresh::OnFirstRead) - { - let scope = Scope::Query { - evaluation_time_ms: evaluation_ms, - }; - refreshes.push((vec![view], scope)); - } - let raw_data_reads_before = RAW_DATA_READS.with(Cell::get); - for (members, scope) in refreshes { - // The views of the run whose state the store lacks, one per state. - let mut roots: Vec<&MaterializedView> = Vec::new(); - for view in members { - let key = &planned[&(view.node_id as u64)].0; - if !store.contains_key(key) - && !roots - .iter() - .any(|root| &planned[&(root.node_id as u64)].0 == key) - { - roots.push(view); - } - } - if roots.is_empty() { - continue; - } - // The views the run reads that it does not refresh, from the - // store; its raw data is bound at its `Scan`s. - let mut sources = BTreeMap::new(); - for id in roots - .iter() - .flat_map(|view| &view.inputs.materialized_views) - { - if !roots.iter().any(|root| root.node_id == *id) { - let key = &planned[&(*id as u64)].0; - sources.insert(*id as u64, store[key].state.clone()); - } - } - let root_ids: Vec = roots.iter().map(|view| view.node_id as u64).collect(); - // Each root's batches, in one batch per root, as the store keeps it. - let outputs = run(dag, sources, samples, &root_ids, scope); - for (view, batches) in roots.into_iter().zip(outputs) { - let (key, span) = &planned[&(view.node_id as u64)]; - let schema = batches[0].schema().clone(); - let state_rows = batches.iter().flat_map(|b| b.to_rows().to_vec()).collect(); - store.insert( - key.clone(), - StoredState { - state: Batch::try_new(schema, state_rows).unwrap(), - start_ms: span.0, - retention_ms: view.retention_ms(), - }, - ); - outputs_built += 1; - } - } - let raw_data_reads_refreshing = RAW_DATA_READS.with(Cell::get) - raw_data_reads_before; - // Each view's state and the span of its data, by node id. - let mut built: BTreeMap = BTreeMap::new(); - for view in &views { - let (key, span) = &planned[&(view.node_id as u64)]; - // One stored state serves every view with its key, so it is kept - // for the largest retention any of them requires. - let stored = store.get_mut(key).unwrap(); - stored.retention_ms = stored.retention_ms.max(view.retention_ms()); - built.insert(view.node_id as u64, (stored.state.clone(), *span)); - } - // The query run reads the views it reaches from the queries' roots - // without passing another view; those below are read only through the - // views refreshed from them. - let mut pending = query_roots(&installed); - let mut seen = std::collections::BTreeSet::new(); - while let Some(id) = pending.pop() { - if !seen.insert(id) { - continue; - } - match built.get(&id) { - Some((state, _)) => { - inputs.insert(id, state.clone()); - } - None => pending.extend( - dag.edges - .iter() - .filter(|edge| edge.consumer as u64 == id) - .map(|edge| edge.producer as u64), - ), - } - } - // The query evaluation, which reads the stored states. - let query = Scope::Query { - evaluation_time_ms: evaluation_ms, - }; - // Each query's answers: (series labels, value) per series. - let answers_per_query = run( - dag, - inputs, - samples, - &query_roots(&installed), - query.clone(), - ) - .iter() - .map(|batches| answers(batches)) - .collect(); - // The store drops each state whose data starts more than its retention - // before this evaluation. - store.retain(|_, state| state.start_ms >= evaluation_ms - state.retention_ms as i64); - - // The correctness reference: nothing stored, every window read from - // the raw samples at query time. - let reference = asap_planner::test_support::query_time_reference(&installed).unwrap(); - // The reference's DAG. - let reference_dag = &reference.dag; - // The reference's answers, which the installed plan's must equal. - let reference_answers_per_query = run( - reference_dag, - BTreeMap::new(), - samples, - &query_roots(&reference), - query, - ) - .iter() - .map(|batches| answers(batches)) - .collect(); - DataPlaneRun { - answers_per_query, - reference_answers_per_query, - outputs_built, - raw_data_reads_refreshing, - } -} - -/// The sketch algorithm of the tumbling windows view `id` of `plan` builds. -fn tumbling_window_sketch(plan: &InstallablePlan, id: usize) -> SketchAlgorithm { - let node = plan.dag.nodes.iter().find(|node| node.id == id).unwrap(); - match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => *kind.algorithm(), - other => panic!("not a sketch tumbling window: {other:?}"), - } -} - -/// Issue #1151's example: a p90 at rank error 0.01 over a minute and a p99 -/// at relative error 0.01 over five minutes, both every 10 s, over 1,000 -/// ingesting non-negative `latency_ms` gauges sampled every 10 s, on a -/// deployment that keeps no raw data: a KLL chain and a DDSketch chain, -/// both of 10 s tumbling windows, kept at ingestion over one `Scan`. Their tumbling windows are one refresh group, so each -/// 10 s window is read from raw data once, in one ingestion run that builds -/// both tumbling windows ending there: 30 reads for the 30 tumbling window windows of the first -/// evaluation (36 when each tumbling window view had its own run). Each state equals -/// the one a run of its view alone builds, and both queries answer as the -/// query-time reference. -#[tokio::test] -async fn tumbling_windows_of_one_refresh_window_read_each_raw_window_once() { - let entry = |query: &str, metric: &str| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {{"accuracy_metric": {metric:?}, "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }; - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}, {}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 100.0}}}}, - "sources": {{"latency_ms": {{"series_count": 1000, "sample_interval": 10000, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, - entry("quantile_over_time(0.9, latency_ms[1m])", "rank"), - entry("quantile_over_time(0.99, latency_ms[5m])", "relative"), - )) - .unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - installed.check_compiles().unwrap(); - // One refresh group: the six 10 s KLL tumbling windows and the thirty 10 s - // DDSketch tumbling windows. - let groups = installed.ingestion_refresh_groups().unwrap(); - assert_eq!(groups.len(), 1, "{groups:#?}"); - assert_eq!( - groups[0].refresh_window, - Some(RefreshWindow { - width_ms: 10_000, - phase_ms: 0 - }) - ); - let sketches: Vec = groups[0] - .views - .iter() - .map(|view| tumbling_window_sketch(&installed, view.node_id)) - .collect(); - let count = |algorithm| sketches.iter().filter(|a| **a == algorithm).count(); - assert_eq!(count(SketchAlgorithm::Kll), 6, "{sketches:?}"); - assert_eq!(count(SketchAlgorithm::DDSketch), 30, "{sketches:?}"); - - let samples = remote_write_samples(); - let mut store = SummaryStore::new(); - let evaluated = run_data_plane(&plan_json, &samples, EVALUATION_MS, &mut store); - assert_eq!(evaluated.outputs_built, 36); - assert_eq!(evaluated.raw_data_reads_refreshing, 30); - assert_eq!( - evaluated.answers_per_query, - evaluated.reference_answers_per_query - ); - assert_eq!(evaluated.answers_per_query[0].len(), 2); - - // Each tumbling window's state equals the one an ingestion run of its view alone - // builds. - for view in &groups[0].views { - let span = view.inputs.raw_data_windows()[0].bounds_ms(EVALUATION_MS); - let scope = Scope::Ingestion { - window_start_ms: span.0, - window_end_ms: span.1, - }; - let alone = run( - &installed.dag, - BTreeMap::new(), - &samples, - &[view.node_id as u64], - scope, - ) - .remove(0); - // `Value` has no equality; its debug form shows each state. - let rows = format!( - "{:?}", - alone - .iter() - .flat_map(|b| b.to_rows().to_vec()) - .collect::>() - ); - let stored = &store[&state_key(view, EVALUATION_MS)].state; - assert_eq!( - format!("{:?}", stored.to_rows()), - rows, - "view {}", - view.node_id - ); - } -} - -/// #509 Example 4b, as a runtime runs it: one `plan_installable` call, -/// `plan.json`, then each tumbling window view refreshed in an ingestion run and merged -/// in the query run. One p99 per series, read from the hour's oldest tumbling window, -/// equal to the query-time reference; the other metric is not read. -#[tokio::test] -async fn a_runtime_installs_and_runs_the_plan_from_one_facade_call() { - let plan_json = plan_json(&["quantile_over_time(0.99, latency_ms[60m])"]).await; - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = installed.materialized_views().unwrap(); - assert!(!views.is_empty()); - assert_eq!(run.outputs_built, views.len()); - // The p99 of the hour comes from its oldest tumbling window, tumbling window 5. - assert_answers_from_tumbling_window(&run.answers_per_query[0], 5); - assert_eq!(run.answers_per_query[0][0].0["instance"], "a"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - /// The plan file carries Stage 3's predicted usage of the selected plan, so /// a runtime reports it against what it measures without planning again: /// for Example 4b's tumbling windows maintained at ingestion time, its nodes' prices @@ -795,392 +278,6 @@ async fn the_plan_carries_stage3s_predicted_usage() { .any(|node| node.installed_node_ids == [root])); } -/// The query's label matcher selects series `a` only: the runtime feeds -/// every sample, and the window's selector applies the plan's `Scan`. -#[tokio::test] -async fn label_matchers_select_the_series_a_tumbling_window_reads() { - let plan_json = - plan_json(&[r#"quantile_over_time(0.99, latency_ms{instance="a"}[60m])"#]).await; - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), 1, "{answered:?}"); - assert_eq!(answered[0].0["instance"], "a"); - assert_answers_from_tumbling_window(answered, 5); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// The p99 over the last hour and over the hour before the last 10 -/// minutes may share selected tumbling windows. The runtime refreshes each distinct -/// state key once, independent of the optimizer's tumbling-window widths. Both answers -/// equal the reference. -#[tokio::test] -async fn an_hour_and_its_offset_twin_both_equal_the_reference() { - let plan_json = plan_json(&[ - "quantile_over_time(0.99, latency_ms[60m])", - "quantile_over_time(0.99, latency_ms[60m] offset 10m)", - ]) - .await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let distinct_states: BTreeSet<_> = installed - .materialized_views() - .unwrap() - .iter() - .map(|view| state_key(view, EVALUATION_MS)) - .collect(); - assert!(!distinct_states.is_empty()); - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, distinct_states.len()); - assert_eq!(run.answers_per_query.len(), 2); - // The last hour's p99 comes from tumbling window 5, the offset hour's from tumbling window 6. - assert_answers_from_tumbling_window(&run.answers_per_query[0], 5); - assert_answers_from_tumbling_window(&run.answers_per_query[1], 6); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Two copies of `sum by (job) (sum_over_time(m[10m]))`, each every 20 s, -/// one at phase 0 and one at phase 5 s, over 6 ingesting series of `m` -/// sampled every second. Tumbling windows shared by both must end on both schedules: -/// 20 s tumbling windows end only on the first's, so the second would read -/// `(t − 605 s, t − 5 s]` instead of `(t − 600 s, t]`. -#[tokio::test] -async fn queries_at_two_phases_read_tumbling_windows_on_both_schedules() { - assert_phased_copies_answer_as_reference(&[0, 5_000]).await; -} - -/// The same two copies, both at phase 5 s. A runtime anchors tumbling windows at the -/// epoch and the plan carries no tumbling-window origin, so 20 s tumbling windows would end 5 s -/// before every evaluation: the tumbling windows must also end on the phase itself. -#[tokio::test] -async fn queries_at_one_nonzero_phase_read_tumbling_windows_anchored_at_the_epoch() { - assert_phased_copies_answer_as_reference(&[5_000, 5_000]).await; -} - -/// Copies of `sum by (job) (sum_over_time(m[10m]))`, each every 20 s, copy -/// `q` at phase `phases_ms[q]`, over 6 ingesting series of `m` sampled -/// every second. Run by a runtime that refreshes each ingestion tumbling window once, on -/// the grid of its width anchored at the epoch, each answers as the -/// query-time reference at its own evaluation. -async fn assert_phased_copies_answer_as_reference(phases_ms: &[i64]) { - let entries: Vec = phases_ms - .iter() - .map(|phase_ms| { - format!( - r#"{{"query": "sum by (job) (sum_over_time(m[10m]))", - "demand": {{"fixed_interval_at": {{"interval": 20000, "evaluation_phase": {phase_ms}}}}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 6.0}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap(); - // A deployment that keeps no raw data samples: the windows are views. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - assert!( - views - .iter() - .any(|view| view.refresh() == MaterializedViewRefresh::AtIngestion), - "{views:#?}" - ); - // Samples of `m`, `job` a (instances 1-3) and b (4-6), every second - // from 11 minutes before the first evaluation to 10 s after it; each - // value distinct, so a window off by 5 s at either edge answers - // differently. - let samples: Vec = (1..=6i64) - .flat_map(|instance| { - (0..680i64).map(move |step| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 660_000 + step * 1_000, - value: (instance * 1000 + step * step % 997) as f64, - }) - }) - .collect(); - // The tumbling windows stored across every evaluation. - let mut store = SummaryStore::new(); - // Copy `q` evaluates `phases_ms[q]` after `EVALUATION_MS` (on the - // 20 s grid); each reads the tumbling windows the store holds by then. - for (query, phase_ms) in phases_ms.iter().enumerate() { - let evaluation_ms = EVALUATION_MS + phase_ms; - let run = run_data_plane_on( - &plan_json, - &samples, - evaluation_ms, - &mut store, - IngestionBuilds::Tumbling, - ); - assert_eq!(run.answers_per_query[query].len(), 2, "one sum per job"); - assert_eq!( - run.answers_per_query[query], run.reference_answers_per_query[query], - "query {query} at {evaluation_ms}" - ); - } -} - -/// A query whose window equals its interval, with no declared phase: one -/// tumbling window per evaluation, maintained at ingestion time, answers -/// every evaluation without raw data. `quantile_over_time(0.99, m[5m])` -/// every 5 minutes at rank error 0.01 (a KLL per series) had no plan. -#[tokio::test] -async fn a_window_equal_to_its_interval_is_kept_at_ingestion_time() { - assert_answers_as_reference_without_raw_data( - "quantile_over_time(0.99, m[5m])", - r#"{"fixed_interval": 300000}"#, - RANK_ERROR_1_PERCENT, - &[0, 300_000, 600_000], - ) - .await; - assert_answers_as_reference_without_raw_data( - "sum by (job) (sum_over_time(m[20s]))", - r#"{"fixed_interval": 20000}"#, - "{}", - &[0, 20_000, 40_000, 60_000], - ) - .await; -} - -/// The same 20 s window every 20 s at phase 5 s: a 20 s window on the -/// epoch grid would end 5 s before every evaluation, so the plan must not -/// read one; it still answers every evaluation as the reference. -#[tokio::test] -async fn a_window_equal_to_its_interval_at_an_offset_phase_answers_as_reference() { - assert_answers_as_reference_without_raw_data( - "sum by (job) (sum_over_time(m[20s]))", - r#"{"fixed_interval_at": {"interval": 20000, "evaluation_phase": 5000}}"#, - "{}", - &[5_000, 25_000, 45_000, 65_000], - ) - .await; -} - -/// Rank error 0.01 with failure probability 0.01, as `requirements`. -const RANK_ERROR_1_PERCENT: &str = r#"{"accuracy": {"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}}"#; - -/// Plans `query` with `demand` and `requirements` over 6 ingesting series of `m` sampled every -/// second, for a deployment that keeps no raw data, and runs it at -/// `EVALUATION_MS` plus each of `evaluation_offsets_ms` on a runtime that -/// refreshes each ingestion state once, on the epoch grid of its width. The -/// plan keeps state at ingestion time and reads no raw data at query time; -/// every answer equals the query-time reference. -async fn assert_answers_as_reference_without_raw_data( - query: &str, - demand: &str, - requirements: &str, - evaluation_offsets_ms: &[i64], -) { - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [ - {{"query": {query:?}, "demand": {demand}, "requirements": {requirements}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 6.0}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 1000}}}}}}}}"# - )) - .unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - assert!( - views - .iter() - .any(|view| view.refresh() == MaterializedViewRefresh::AtIngestion), - "{views:#?}" - ); - assert!( - plan.raw_data_retention().unwrap().is_empty(), - "no raw data at query time" - ); - // Samples of `m`, `job` a (instances 1-3) and b (4-6), every second - // from 6 minutes before `EVALUATION_MS` to past the last evaluation; - // each value distinct, so a window off by a second answers differently. - let last_ms = EVALUATION_MS + evaluation_offsets_ms.iter().max().unwrap(); - let steps = (last_ms - EVALUATION_MS + 360_000) / 1_000 + 10; - let samples: Vec = (1..=6i64) - .flat_map(|instance| { - (0..steps).map(move |step| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 360_000 + step * 1_000, - value: (instance * 1000 + step * step % 997) as f64, - }) - }) - .collect(); - // The states stored across every evaluation. - let mut store = SummaryStore::new(); - for offset_ms in evaluation_offsets_ms { - let evaluation_ms = EVALUATION_MS + offset_ms; - let run = run_data_plane_on( - &plan_json, - &samples, - evaluation_ms, - &mut store, - IngestionBuilds::Tumbling, - ); - assert!(!run.answers_per_query[0].is_empty()); - assert_eq!( - run.answers_per_query, run.reference_answers_per_query, - "{query} at {evaluation_ms}" - ); - } -} - -/// The same exact query twice, one copy every 10 s and one every 15 s -/// (hash-consed into one memo group), on a deployment that keeps no raw -/// data. A tumbling-window chain is built at ingestion time once per -/// tumbling window, on the grid of its width anchored at the epoch, so a -/// chain either copy reads must have a width dividing that copy's interval: -/// a 15 s chain read every 10 s ends up to 10 s before the evaluation -/// (#1173). Every evaluation of both copies over one minute answers as the -/// query-time reference. -#[tokio::test] -async fn queries_at_two_intervals_read_chains_on_their_own_grids() { - for query in [ - "sum by (job) (rate(m[1m]))", - "sum by (job) (sum_over_time(m[1m]))", - ] { - assert_copies_at_intervals_answer_as_reference(query, &[10_000, 15_000]).await; - } -} - -/// Copies of `query`, copy `q` every `intervals_ms[q]`, over 6 ingesting -/// series of `m` sampled every second, on a deployment that keeps no raw -/// data. Run by a runtime that refreshes each ingestion tumbling window -/// once, on the grid of its width anchored at the epoch, every evaluation -/// of every copy over one minute answers as the query-time reference. -async fn assert_copies_at_intervals_answer_as_reference(query: &str, intervals_ms: &[i64]) { - let entries: Vec = intervals_ms - .iter() - .map(|interval_ms| { - format!( - r#"{{"query": {query:?}, - "demand": {{"fixed_interval": {interval_ms}}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 6.0}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap(); - // A deployment that keeps no raw data samples: the windows are views. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - // Samples of `m`, `job` a (instances 1-3) and b (4-6), every second - // from 2 minutes before `EVALUATION_MS` to past the last evaluation; - // each value distinct, so a window off by a second answers differently. - let samples: Vec = (1..=6i64) - .flat_map(|instance| { - (0..200i64).map(move |step| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 120_000 + step * 1_000, - value: (instance * 1000 + step * step % 997) as f64, - }) - }) - .collect(); - // Every evaluation of every copy over one minute, in time order. - let mut evaluations: Vec<(usize, i64)> = intervals_ms - .iter() - .enumerate() - .flat_map(|(query, &interval_ms)| { - (0..=60_000 / interval_ms).map(move |k| (query, EVALUATION_MS + k * interval_ms)) - }) - .collect(); - evaluations.sort_by_key(|&(query, evaluation_ms)| (evaluation_ms, query)); - // The states stored across every evaluation. - let mut store = SummaryStore::new(); - for (copy, evaluation_ms) in evaluations { - let run = run_data_plane_on( - &plan_json, - &samples, - evaluation_ms, - &mut store, - IngestionBuilds::Tumbling, - ); - assert_eq!(run.answers_per_query[copy].len(), 2, "one answer per job"); - assert_eq!( - run.answers_per_query[copy], run.reference_answers_per_query[copy], - "{query}, copy {copy} (every {} ms) at {evaluation_ms}", - intervals_ms[copy] - ); - } -} - -/// A plan file of another DAG wire version is refused, and so is the -/// reference of a tampered plan file, with an error rather than a panic. -#[tokio::test] -async fn a_plan_file_of_another_version_or_tampered_is_refused() { - let plan_json = plan_json(&["quantile_over_time(0.99, latency_ms[60m])"]).await; - let mut other_version: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - other_version.wire_version += 1; - assert!(matches!( - other_version.validate(), - Err(InstallablePlanError::UnsupportedWireVersion(_)) - )); - let mut tampered: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - tampered.query_texts.clear(); - assert!(matches!( - asap_planner::test_support::query_time_reference(&tampered), - Err(InstallablePlanError::QueriesDoNotMatchRoots { .. }) - )); - let mut missing_node: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - missing_node.dag.nodes.remove(0); - assert!(matches!( - asap_planner::test_support::query_time_reference(&missing_node), - Err(InstallablePlanError::InvalidDag(_)) - )); -} - /// `sum(latency_ms)` reads an instant selector: per series, only the latest /// sample in its 1 s lookback counts. Series `a` has samples at T-900 ms (1) /// and T-100 ms (10), series `b` at T-200 ms (100): the sum is 10 + 100. @@ -1217,1518 +314,59 @@ async fn an_instant_selector_reads_the_latest_sample_per_series() { assert_eq!(answered[0].1, 110.0); } -/// A deployment that keeps raw data samples and query-time results across -/// evaluations (#509 Example 4's B3), with a dashboard panel -/// (`max_over_time(latency_ms[5m])` every minute over 1M series sampled -/// every 1.5 s, answered as the last hour at 1-minute steps): Stage 3 -/// keeps five 1-minute tumbling windows refreshed at query time, and keeps raw data -/// for refreshing the oldest tumbling window at the oldest step (5 min + 1 h). The first -/// evaluation, with an empty store, refreshes all five in its query run; the -/// next, a minute later, reuses the four still in its window and refreshes -/// only the newest. Each -/// evaluation's answers (each series' newest-tumbling-window maximum) equal the -/// reference. +// ── A ratio of two relative-error quantiles (issue #852) ──────────────────── + +// ── Materialized views refreshed from materialized views (issue #781) ────────────────── + +/// The MVP benchmark's p99 query: the per-series p99 of a gauge over 100 s, +/// averaged per (region, service, job), every second, within 0.05 at +/// failure probability 0.3, over 1,000 `data` series sampled every 100 ms, +/// with `data` declared non-negative or not. +fn p99_avg_workload(values_non_negative: bool) -> PlanningWorkload { + serde_json::from_str(&format!( + r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ + "query": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "demand": {{"fixed_interval": 1000}}, + "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3, + "accuracy_scope": "whole_result"}}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 10000.0}}}}, + "sources": {{"data": {{"series_count": 1000, "metric_type": "gauge", + "sample_interval": 100, "column_signs": {{"value": "{sign}"}}}}}}}}}}"#, + sign = if values_non_negative { + "non_negative" + } else { + "unknown" + } + )) + .unwrap() +} + +/// The MVP benchmark's p99 query over an undeclared gauge, which has no +/// valid plan with the built-in model, gets one with +/// `ignore_accuracy_guarantees`. #[tokio::test] -async fn query_time_results_kept_across_evaluations_are_built_in_query_runs() { +async fn ignoring_accuracy_guarantees_plans_the_undeclared_p99_query() { let capabilities = DeploymentCapabilities { - query_time_results_retained: true, + raw_data_retained: false, ..DeploymentCapabilities::default() }; - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "max_over_time(latency_ms[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}, - "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 666666.67}}, - "sources": {"latency_ms": {"series_count": 1000000, "sample_interval": 1500}}}}"#, - ) - .unwrap(); - let plan_json = plan_json_for(workload, &capabilities).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = installed.materialized_views().unwrap(); - assert_eq!(views.len(), 5); - assert!(views - .iter() - .all(|v| v.refresh() == MaterializedViewRefresh::OnFirstRead - && v.inputs.raw_data_windows()[0].width_ms() == 60_000)); - // Raw data is kept for refreshing the oldest tumbling window at the oldest step: - // the five tumbling windows' 5 minutes plus the hour of steps, as long as the tumbling windows - // themselves are kept. - let tumbling_window_lookback_ms = views - .iter() - .flat_map(|v| v.inputs.raw_data_windows()) - .map(asap_types::test_support::lookback_ms) - .max(); - assert_eq!(tumbling_window_lookback_ms, Some(300_000)); - assert!(views.iter().all(|v| v.retention_ms() == 3_900_000)); - let retention = installed.raw_data_retention().unwrap(); - assert_eq!( - retention - .iter() - .map(|r| r.lookback_ms()) - .collect::>(), - vec![Some(3_900_000)], - "{retention:?}" - ); - // The tumbling window cache the data plane keeps across evaluations. - let mut store = SummaryStore::new(); - let first = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut store, - ); - assert_eq!(first.outputs_built, 5); - assert_answers_from_tumbling_window(&first.answers_per_query[0], 0); - assert_eq!(first.answers_per_query, first.reference_answers_per_query); - let next = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS + 60_000, - &mut store, - ); - assert_eq!( - next.outputs_built, 1, - "four tumbling windows reused from the cache" + let settings = CostInputs::default(); + let (plan, report) = plan_installable(UserInput::new( + &p99_avg_workload(false), + FrontendInput::Promql, + PlanningModels::builtin() + .with_deployment_capabilities(&capabilities) + .with_cost_inputs(&settings) + .with_ignore_accuracy_guarantees(true), + )) + .await + .expect("a plan is selected"); + report.as_ref().expect("the plan's cost report"); + assert!( + plan.raw_data_retention().unwrap().is_empty(), + "no raw samples read" ); - assert_eq!(next.answers_per_query, next.reference_answers_per_query); -} - -/// An exact per-series quantile under a group average (issue #770), on a -/// deployment that keeps raw data samples: Stage 3 selects the exact plan, which -/// reads each series' raw data samples in the window at query time, and the -/// executor runs it. Series `a` (1..=10) and `b` (10, 20, ..., 100) are in -/// `east`, `c` (constantly 5) in `west`; a sample of each before the window -/// is not read. Per series, p99 interpolates between its two largest -/// samples (rank 0.99 × 9 = 8.91): 9.91 for `a`, 99.1 for `b`, 5 for `c`. -#[tokio::test] -async fn an_exact_per_series_quantile_runs_from_raw_data_samples() { - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "avg by (region) (quantile_over_time(0.99, latency_ms[100s]))", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 3.0}}, - "sources": {"latency_ms": {"series_count": 3, "sample_interval": 10000}}}}"#, - ) - .unwrap(); - let plan_json = plan_json_for(workload, &DeploymentCapabilities::default()).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert!( - installed.materialized_views().unwrap().is_empty(), - "the exact plan" - ); - assert_eq!(installed.raw_data_retention().unwrap().len(), 1); - // Ten samples per series in the window, one every 10 s, and one before it. - let mut samples = Vec::new(); - for (instance, region, value) in [ - ("a", "east", (|step| step as f64) as fn(i64) -> f64), - ("b", "east", |step| 10.0 * step as f64), - ("c", "west", |_| 5.0), - ] { - for step in 0..=10 { - let mut sample = sample( - "latency_ms", - instance, - EVALUATION_MS - 105_000 + 10_000 * step, - if step == 0 { 1e9 } else { value(step) }, - ); - sample - .labels - .insert("region".to_string(), region.to_string()); - samples.push(sample); - } - } - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), 2, "{answered:?}"); - for ((labels, value), (region, expected)) in answered - .iter() - .zip([("east", (9.91 + 99.1) / 2.0), ("west", 5.0)]) - { - assert_eq!(labels["region"], region); - assert!((value - expected).abs() < 1e-9, "{region}: {value}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -// ── A ratio of two relative-error quantiles (issue #852) ──────────────────── - -/// Repeating `queries`, each every 10 minutes at error 0.05 in each -/// summary's own metric (W-RangeHeavy leaves the metric unset) and δ 0.01, -/// over 1,000 ingesting non-negative `latency_ms` gauges sampled -/// every second. -fn relative_workload(queries: &[&str]) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 600000}}, - "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.01, - "accuracy_scope": "whole_result"}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, - "sources": {{"latency_ms": {{"series_count": 1000, "sample_interval": 1000, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// The DDSketch α of every sketch `plan` builds; panics on any other sketch. -fn ddsketch_alphas(plan: &InstallablePlan) -> Vec { - plan.dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => match kind.params() { - SketchParams::DDSketch { alpha } => Some(*alpha), - other => panic!("not a DDSketch: {other:?}"), - }, - _ => None, - }) - .collect() -} - -/// PromQL `quantile_over_time(q, …)` of `values`: linear interpolation -/// between the sorted values at rank `q·(n − 1)`. -fn promql_quantile(q: f64, values: &mut [f64]) -> f64 { - values.sort_by(f64::total_cmp); - let rank = q * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - values[low] + (values[high] - values[low]) * (rank - low as f64) -} - -/// The p90/p50 ratio of an hour (W-RangeHeavy template 8) on a deployment -/// that keeps no raw data samples: both quantiles come from DDSketches -/// sized for a share of the target (α = 1 − 1.05^{−1/2} ≈ 0.0241, so -/// `(a + b)/(1 − b) ≤ 0.05`), maintained in tumbling windows; nothing is read from -/// raw data at query time. Run on data, each series' ratio is within 0.05 -/// of the exact ratio of its hour's samples. -#[tokio::test] -async fn a_ratio_of_two_quantiles_runs_from_ddsketches_within_its_relative_target() { - let query = - "quantile_over_time(0.9, latency_ms[60m]) / quantile_over_time(0.5, latency_ms[60m])"; - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(relative_workload(&[query]), &capabilities).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert!( - installed.raw_data_retention().unwrap().is_empty(), - "no raw scan" - ); - let alphas = ddsketch_alphas(&installed); - assert!(!alphas.is_empty(), "DDSketch states"); - let share = 1.0 - 1.05f64.powf(-0.5); - for alpha in &alphas { - assert!(*alpha <= share + 1e-12, "α {alpha} > {share}"); - } - let samples = remote_write_samples(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), 2, "{answered:?}"); - for (labels, value) in answered { - let mut hour: Vec = samples - .iter() - .filter(|s| { - s.labels["__name__"] == "latency_ms" - && s.labels["instance"] == labels["instance"] - && s.timestamp_ms > EVALUATION_MS - 3_600_000 - && s.timestamp_ms <= EVALUATION_MS - }) - .map(|s| s.value) - .collect(); - let exact = promql_quantile(0.9, &mut hour) / promql_quantile(0.5, &mut hour); - let error = (value - exact).abs() / exact; - assert!(error <= 0.05, "{labels:?}: {value} vs {exact} ({error})"); - } -} - -/// The ratio's numerator is also asked for on its own (W-RangeHeavy -/// templates 5 and 8): the p90 estimate is shared, so its state is sized -/// for its strictest reader, the ratio's share; the plan still reads no raw -/// data samples. -#[tokio::test] -#[ignore = "slow without a planning budget: over 30 min in a debug build; #1197"] -async fn a_quantile_shared_with_a_ratio_is_sized_for_the_ratio() { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for( - relative_workload(&[ - "quantile_over_time(0.9, latency_ms[60m])", - "quantile_over_time(0.9, latency_ms[60m]) / quantile_over_time(0.5, latency_ms[60m])", - ]), - &capabilities, - ) - .await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert!( - installed.raw_data_retention().unwrap().is_empty(), - "no raw scan" - ); - let share = 1.0 - 1.05f64.powf(-0.5); - for alpha in ddsketch_alphas(&installed) { - assert!(alpha <= share + 1e-12, "α {alpha} > {share}"); - } - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -// ── Materialized views refreshed from materialized views (issue #781) ────────────────── - -/// Stage 1 to Stage 3, then every node `store` names stored as it says, -/// the others as Stage 2 chose, and the plan timed again: how a test makes -/// the plan store a shape Stage 3 does not select today. -struct StoredAsForced { - store: fn(&OperatorNode) -> Option, -} - -impl OptimizationPass for StoredAsForced { - fn name(&self) -> &'static str { - "stored-as-forced" - } - - fn optimize(&self, input: OptimizationInput<'_>) -> Result { - let mut output = Stage1ToStage3Pass.optimize(input)?; - // Each node untimed once, so shared inputs stay shared, and its - // materialization in `assignment`. - let mut untimed: HashMap<*const OperatorNode, Rc> = HashMap::new(); - let mut assignment = MaterializationAssignment::default(); - let roots: Vec> = output - .plans - .iter() - .map(|plan| untime(&plan.root, self.store, &mut untimed, &mut assignment)) - .collect(); - // A node read both at ingestion and at query time is copied, one - // copy per timing, as Stage 2 does. - let (roots, assignment) = split_shared_by_timing(&roots, &assignment); - let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - for (plan, root) in output.plans.iter_mut().zip(timed) { - let QueryRoot::Operator(root) = root else { - unreachable!("operator roots in, operator roots out") - }; - plan.root = root; - } - Ok(output) - } -} - -/// `node` untimed, with `store`'s materialization (or its own) recorded in -/// `assignment` for every node of its sub-DAG. -fn untime( - node: &Rc, - store: fn(&OperatorNode) -> Option, - untimed: &mut HashMap<*const OperatorNode, Rc>, - assignment: &mut MaterializationAssignment, -) -> Rc { - if let Some(done) = untimed.get(&Rc::as_ptr(node)) { - return Rc::clone(done); - } - let mut copy = (**node).clone(); - copy.operator = node - .operator - .map_children(|child| untime(child, store, untimed, assignment)); - copy.timing = None; - copy.materialization = Materialization::NotMaterialized; - let copy = Rc::new(copy); - // A shared non-summary node Stage 2 built once per evaluation is not - // kept: a build forced to ingestion time above it cannot read a - // query-time result. - let own = match node.asap() { - Some( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryTumblingWindowChain { .. } - | ASAPOp::SummaryMerge { .. }, - ) => node.materialization, - _ => Materialization::NotMaterialized, - }; - let materialization = store(node).unwrap_or(own); - if materialization != Materialization::NotMaterialized { - assignment.set(©, materialization); - } - untimed.insert(Rc::as_ptr(node), Rc::clone(©)); - copy -} - -/// Every `SummaryAgg` built at ingestion time. -fn summaries_at_ingestion(node: &OperatorNode) -> Option { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. })) - .then_some(Materialization::MatAtIngestion(Medium::Memory)) -} - -/// Every `SummaryMerge` built at query time and kept across evaluations. -fn merges_kept(node: &OperatorNode) -> Option { - matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. })).then_some( - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ) -} - -/// Exact PromQL `queries`, every minute, over 6 ingesting series of `m` -/// sampled every 10 s; `tail` is appended to each entry's JSON (e.g. its -/// range evaluation). -fn exact_workload(queries: &[&str], tail: &str) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 60000}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}{tail}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 0.6}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 10000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// The plan file of `workload` on `capabilities`, with what `store` names -/// stored. -async fn forced_plan_json( - workload: &PlanningWorkload, - capabilities: &DeploymentCapabilities, - store: fn(&OperatorNode) -> Option, -) -> String { - let pass = StoredAsForced { store }; - let mut input = UserInput::new( - workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(capabilities), - ); - input.pass = Some(&pass); - serde_json::to_string(&plan_installable(input).await.unwrap().0).unwrap() -} - -/// Samples of `m` for six series, `job` a (instances 1, 2, 3) and b (4, 5, -/// 6), every 10 s over the last 10 minutes; each value distinct. -fn job_samples() -> Vec { - let mut samples = Vec::new(); - for instance in 1..=6 { - for step in 0..60 { - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 595_000 + step * 10_000, - value: (instance * 1000 + (step * 37) % 101) as f64, - }); - } - } - samples -} - -/// The time selection `(from_ms, to_ms]` relative to the evaluation. -fn relative_time(from_ms: i64, to_ms: i64) -> Vec { - vec![SelectionBox::new( - Default::default(), - Some((Bound::Excluded(from_ms), Bound::Included(to_ms))), - )] -} - -/// A per-group maximum and a per-group sum (`max by (job)` and -/// `sum by (job, instance)`), both refreshed at ingestion from one stored -/// per-series sum: Stage 1 shares the per-series sums the two queries read -/// (over scans that project other labels). The per-series sum is listed -/// first, then each group output, which reads only it, with its time -/// selection under a definition of its own; refreshed in that order from -/// stored inputs, they answer as the query-time reference. (A coarse sum -/// in place of the maximum rolls up from the fine sum instead: -/// [`a_coarse_sum_rolls_up_from_the_stored_fine_sum`].) -#[tokio::test] -async fn group_outputs_from_one_stored_per_series_sum_are_listed_and_built() { - let workload = exact_workload( - &[ - "max by (job) (sum_over_time(m[1m]))", - "sum by (job, instance) (sum_over_time(m[1m]))", - ], - "", - ); - let plan_json = forced_plan_json( - &workload, - &DeploymentCapabilities::default(), - summaries_at_ingestion, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let [per_series, sums @ ..] = views.as_slice() else { - panic!("{:#?}", views); - }; - assert_eq!(sums.len(), 2, "{:#?}", views); - assert!(per_series.inputs.materialized_views.is_empty()); - assert_eq!(per_series.inputs.raw_data_windows().len(), 1); - for sum in sums { - assert_eq!(sum.inputs.materialized_views, [per_series.node_id]); - assert!(sum.inputs.raw_data_windows().is_empty()); - assert_ne!( - sum.coverage().definition_id(), - per_series.coverage().definition_id() - ); - } - assert_ne!( - sums[0].coverage().definition_id(), - sums[1].coverage().definition_id() - ); - for output in &views { - assert_eq!(output.refresh(), MaterializedViewRefresh::AtIngestion); - assert_eq!(output.coverage().selection(), relative_time(-60_000, 0)); - assert_eq!(output.retention_ms(), 60_000); - } - assert!(plan.raw_data_retention().unwrap().is_empty()); - let run = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, 3); - assert_eq!(run.answers_per_query[0].len(), 2, "one maximum per job"); - assert_eq!(run.answers_per_query[1].len(), 6, "one sum per series"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// A coarse and a fine sum (`sum by (job)` and `sum by (job, instance)`): -/// the fine grouping is priced at no more than the 6 series, as the -/// per-series sum is, so the coarse sum rolls up from it and the plan -/// stores two outputs, the second reading only the first. Built in that -/// order, they answer as the query-time reference. -#[tokio::test] -async fn a_coarse_sum_rolls_up_from_the_stored_fine_sum() { - let workload = exact_workload( - &[ - "sum by (job) (sum_over_time(m[1m]))", - "sum by (job, instance) (sum_over_time(m[1m]))", - ], - "", - ); - let plan_json = forced_plan_json( - &workload, - &DeploymentCapabilities::default(), - summaries_at_ingestion, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let [first, rolled_up] = views.as_slice() else { - panic!("{:#?}", views); - }; - assert!(first.inputs.materialized_views.is_empty()); - assert_eq!(rolled_up.inputs.materialized_views, [first.node_id]); - assert!(rolled_up.inputs.raw_data_windows().is_empty()); - let run = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, 2); - assert_eq!(run.answers_per_query[0].len(), 2, "one sum per job"); - assert_eq!(run.answers_per_query[1].len(), 6, "one sum per series"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// A maximum across series (`max by (job)`) refreshed at ingestion from the -/// stored per-series maxima, which the per-series query reads too: listed -/// after them with their time selection, under a definition of its own. -#[tokio::test] -async fn a_summary_across_series_from_per_series_summaries_is_listed_and_built() { - let workload = exact_workload( - &[ - "max by (job) (max_over_time(m[1m]))", - "max_over_time(m[1m])", - ], - "", - ); - let plan_json = forced_plan_json( - &workload, - &DeploymentCapabilities::default(), - summaries_at_ingestion, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let [per_series, across] = views.as_slice() else { - panic!("{:#?}", views); - }; - assert!(per_series.inputs.materialized_views.is_empty()); - assert_eq!(across.inputs.materialized_views, [per_series.node_id]); - assert!(across.inputs.raw_data_windows().is_empty()); - for output in [per_series, across] { - assert_eq!(output.refresh(), MaterializedViewRefresh::AtIngestion); - assert_eq!(output.coverage().selection(), relative_time(-60_000, 0)); - } - assert_ne!( - across.coverage().definition_id(), - per_series.coverage().definition_id() - ); - let run = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, 2); - assert_eq!(run.answers_per_query[0].len(), 2, "one maximum per job"); - assert_eq!(run.answers_per_query[1].len(), 6, "one maximum per series"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// A dashboard's 5-minute max over the last hour at 1-minute steps (#509 -/// Example 4's B3) keeps five 1-minute tumbling windows; with its merge kept too, the -/// merge is listed after the tumbling windows it reads, with their definition and the -/// union of their times, refreshed in the query run and kept for its window -/// plus the hour of steps; its tumbling windows are kept at least that long. Built -/// from the stored tumbling windows it answers as the reference, and a minute later -/// only the newest tumbling window and the new merge are refreshed. -#[tokio::test] -async fn a_kept_sliding_window_merge_is_listed_and_built() { - let capabilities = DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - }; - let mut workload = exact_workload( - &["max_over_time(m[5m])"], - r#", "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}"#, - ); - // Declared as sampled every second, so that keeping the tumbling windows is - // cheaper than rebuilding the window at every step; the samples the - // run reads are every 10 s, and exact answers do not depend on it. - let facts = workload - .data_workload - .as_mut() - .and_then(|data| data.sources.get_mut("m")) - .unwrap(); - facts.sample_interval = Some(asap_types::workload::DurationMs(1_000)); - let plan_json = forced_plan_json(&workload, &capabilities, merges_kept).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let (merge, tumbling_windows) = views.split_last().unwrap(); - assert_eq!(tumbling_windows.len(), 5); - assert_eq!( - merge.inputs.materialized_views, - tumbling_windows - .iter() - .map(|tumbling_window| tumbling_window.node_id) - .collect::>() - ); - assert!(merge.inputs.raw_data_windows().is_empty()); - assert_eq!(merge.refresh(), MaterializedViewRefresh::OnFirstRead); - assert_eq!(merge.coverage().selection(), relative_time(-300_000, 0)); - assert_eq!(merge.retention_ms(), 300_000 + 3_600_000); - for (i, tumbling_window) in tumbling_windows.iter().enumerate() { - let i = i as i64; - assert_eq!( - tumbling_window.coverage().definition_id(), - merge.coverage().definition_id() - ); - assert_eq!( - tumbling_window.coverage().selection(), - relative_time(-(i + 1) * 60_000, -i * 60_000) - ); - assert!(tumbling_window.retention_ms() >= merge.retention_ms()); - } - let mut store = SummaryStore::new(); - let first = run_data_plane(&plan_json, &job_samples(), EVALUATION_MS, &mut store); - assert_eq!(first.outputs_built, 6); - assert_eq!(first.answers_per_query, first.reference_answers_per_query); - let next = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS + 60_000, - &mut store, - ); - assert_eq!( - next.outputs_built, 2, - "the newest tumbling window and the new merge" - ); - assert_eq!(next.answers_per_query, next.reference_answers_per_query); -} - -/// Example 4b's selected tumbling windows cover the hour and share one definition id, -/// each with its own time selection; each reads its tumbling window of -/// raw data samples and nothing stored. -#[tokio::test] -async fn tumbling_windows_of_an_hour_share_one_definition_with_distinct_times() { - let plan: InstallablePlan = - serde_json::from_str(&plan_json(&["quantile_over_time(0.99, latency_ms[60m])"]).await) - .unwrap(); - let views = plan.materialized_views().unwrap(); - assert!(!views.is_empty()); - let mut tumbling_windows: Vec<&MaterializedView> = views.iter().collect(); - tumbling_windows.sort_by_key(|o| o.inputs.raw_data_windows()[0].end_offset_ms()); - let width_ms = tumbling_windows[0].inputs.raw_data_windows()[0].width_ms(); - assert!(width_ms > 0); - assert_eq!( - width_ms * tumbling_windows.len() as i64, - 3_600_000, - "complete hour coverage" - ); - for (i, tumbling_window) in tumbling_windows.iter().enumerate() { - let i = i as i64; - let [window] = tumbling_window.inputs.raw_data_windows() else { - panic!("{tumbling_window:?}"); - }; - assert_eq!( - (window.width_ms(), window.end_offset_ms()), - (width_ms, i * width_ms) - ); - assert!(tumbling_window.inputs.materialized_views.is_empty()); - assert_eq!( - tumbling_window.coverage().definition_id(), - tumbling_windows[0].coverage().definition_id() - ); - assert_eq!( - tumbling_window.coverage().selection(), - relative_time(-(i + 1) * width_ms, -i * width_ms) - ); - } -} - -/// PromQL `topk` returns whole input series: `topk by (region) (2, -/// sum_over_time(data[1m]))` answers, per region, the two series with the -/// largest sums, each with its own full label set (region, service and -/// instance; `sum_over_time` drops the metric name), not only `region`. -/// Series `e1`..`e3` are in `east` with sums 6, 60 and 30; `w1`, `w2` in -/// `west` with sums 9 and 3. -#[tokio::test] -async fn topk_answers_carry_the_chosen_series_labels() { - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "topk by (region) (2, sum_over_time(data[1m]))", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 0.5}}, - "sources": {"data": {"series_count": 5, "sample_interval": 10000}}}}"#, - ) - .unwrap(); - let plan_json = plan_json_for(workload, &DeploymentCapabilities::default()).await; - let samples = topk_samples(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_full_series_answers(&run, 4); - let labels = |instance: &str, region: &str, service: &str| { - BTreeMap::from([ - ("instance".to_string(), instance.to_string()), - ("region".to_string(), region.to_string()), - ("service".to_string(), service.to_string()), - ]) - }; - let mut answered = run.answers_per_query[0].clone(); - answered.sort_by(|a, b| a.0.cmp(&b.0)); - let mut expected = vec![ - (labels("e2", "east", "web"), 60.0), - (labels("e3", "east", "db"), 30.0), - (labels("w1", "west", "api"), 9.0), - (labels("w2", "west", "web"), 3.0), - ]; - expected.sort_by(|a, b| a.0.cmp(&b.0)); - assert_eq!(answered.len(), expected.len(), "{answered:?}"); - for ((labels, value), (want_labels, want)) in answered.iter().zip(&expected) { - assert_eq!(labels, want_labels); - assert!((value - want).abs() < 1e-9, "{labels:?}: {value}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Samples of `data` for `topk_answers_carry_the_chosen_series_labels`: -/// three per series inside the last minute, each a third of its sum. -fn topk_samples() -> Vec { - let mut samples = Vec::new(); - for (instance, region, service, sum) in [ - ("e1", "east", "api", 6.0), - ("e2", "east", "web", 60.0), - ("e3", "east", "db", 30.0), - ("w1", "west", "api", 9.0), - ("w2", "west", "web", 3.0), - ] { - // In time order, as a counter's samples arrive. - for step in (1..=3).rev() { - let mut sample = sample("data", instance, EVALUATION_MS - step * 15_000, sum / 3.0); - sample.labels.insert("region".into(), region.into()); - sample.labels.insert("service".into(), service.into()); - samples.push(sample); - } - } - samples -} - -/// The first query's `expected` answers, each a whole input series (it -/// carries its `instance`), and the same as the query-time reference's. -fn assert_full_series_answers(run: &DataPlaneRun, expected: usize) { - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), expected, "{answered:?}"); - for (labels, _) in answered { - assert!(labels.contains_key("instance"), "{labels:?}"); - assert!(labels.contains_key("service"), "{labels:?}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Every top-k and bottom-k form, grouped or not, over sums, counts and -/// rates, answers whole input series: `topk(2, ...)` the two of five, -/// `by (region)` per region, `bottomk` likewise. -#[tokio::test] -async fn every_topk_form_answers_whole_series() { - for (query, expected) in [ - ("topk(2, sum_over_time(data[1m]))", 2), - ("topk by (region) (1, count_over_time(data[1m]))", 2), - ("topk by (region) (1, rate(data[1m]))", 2), - ("bottomk by (region) (1, sum_over_time(data[1m]))", 2), - ("bottomk(3, sum_over_time(data[1m]))", 3), - ] { - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ - "query": {query:?}, - "demand": {{"fixed_interval": 60000}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 0.5}}}}, - "sources": {{"data": {{"series_count": 5, "sample_interval": 10000}}}}}}}}"# - )) - .unwrap(); - let plan_json = plan_json_for(workload, &DeploymentCapabilities::default()).await; - let run = run_data_plane( - &plan_json, - &topk_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_full_series_answers(&run, expected); - } -} - -/// The MVP benchmark's p99 query: the per-series p99 of a gauge over 100 s, -/// averaged per (region, service, job), every second, within 0.05 at -/// failure probability 0.3, over 1,000 `data` series sampled every 100 ms, -/// with `data` declared non-negative or not. -fn p99_avg_workload(values_non_negative: bool) -> PlanningWorkload { - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ - "query": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", - "demand": {{"fixed_interval": 1000}}, - "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3, - "accuracy_scope": "whole_result"}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 10000.0}}}}, - "sources": {{"data": {{"series_count": 1000, "metric_type": "gauge", - "sample_interval": 100, "values_non_negative": {values_non_negative}}}}}}}}}"# - )) - .unwrap() -} - -/// [`p99_avg_workload`] planned for a deployment that keeps no raw data samples. -async fn plan_p99_avg( - values_non_negative: bool, -) -> Result<(InstallablePlan, Option), String> { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - plan_installable(UserInput::new( - &p99_avg_workload(values_non_negative), - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .map_err(|error| error.to_string()) -} - -/// Without raw data samples, the only valid plan of the p99 query keeps one -/// DDSketch per series in one-second tumbling windows built at ingestion and merges -/// the window's tumbling windows at query time. A quantile -/// of merged tumbling windows is still one of the samples, so a declared non-negative -/// gauge carries the sketch's relative bound through `avg`: that plan is -/// selected, its answers equal the query-time reference's and are within -/// 5% of the exact averages. -#[tokio::test] -async fn a_non_negative_gauges_merged_ddsketch_tumbling_windows_carry_the_relative_bound_through_avg( -) { - let (plan, selection) = plan_p99_avg(true).await.expect("a valid plan"); - let selection = selection.as_ref().expect("Stage 3's report"); - let views = plan.materialized_views().unwrap(); - assert!(selection.cost_per_second.is_some()); - assert!(!views.is_empty(), "tumbling windows built at ingestion"); - assert_eq!( - stored_definitions(&plan, "DDSketch").len(), - 1, - "one DDSketch" - ); - assert!( - plan.raw_data_retention().unwrap().is_empty(), - "no raw samples read" - ); - // The query's guarantee, with `data`'s declared sign and its 1,000 - // series bound as Stage 3 binds them: a relative error of at most 0.05 - // at failure probability at most 0.3 (DDSketch's bound is - // deterministic). - let root = plan.dag.roots[0]; - let guarantee = plan - .dag - .nodes - .iter() - .find(|node| node.id == root) - .and_then(|node| node.guarantee.clone()) - .expect("the query's guarantee"); - let sign = asap_accuracy::test_support::raw_data_samples_sign_statistic("data"); - let bound = guarantee.bind(&|statistic| match statistic { - "input_row_count" => Some(1_000.0), - _ => (statistic == sign).then_some(1.0), - }); - assert_eq!( - bound.spec.accuracy_metric, - asap_types::ir::properties::AccuracyMetric::Relative - ); - let error = bound.spec.error.evaluate().expect("a numeric bound"); - assert!(error <= 0.05, "{error}"); - let failure = bound - .spec - .failure_probability - .evaluate() - .expect("a numeric failure probability"); - assert!(failure <= 0.3, "{failure}"); - - // Eight non-negative series, one sample every 100 ms over the last - // 110 s: two per (region, service, job) group. - let evaluation_ms = EVALUATION_MS; - let mut samples = Vec::new(); - let mut per_group: BTreeMap<(String, String, String), Vec>> = BTreeMap::new(); - for series in 0..8 { - let region = format!("r{}", series % 2); - let service = format!("s{}", series / 4); - let job = "j".to_string(); - let mut window = Vec::new(); - for step in 0..1_100i64 { - let timestamp_ms = evaluation_ms - 110_000 + 50 + step * 100; - let value = 10.0 * (series + 1) as f64 + ((step * 37 + series * 11) % 997) as f64; - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "data".to_string()), - ("region".to_string(), region.clone()), - ("service".to_string(), service.clone()), - ("job".to_string(), job.clone()), - ("instance".to_string(), format!("i{series}")), - ]), - timestamp_ms, - value, - }); - if timestamp_ms > evaluation_ms - 100_000 { - window.push(value); - } - } - per_group - .entry((region, service, job)) - .or_default() - .push(window); - } - let run = run_data_plane( - &serde_json::to_string(&plan).unwrap(), - &samples, - evaluation_ms, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - let answers = &run.answers_per_query[0]; - assert_eq!(answers.len(), per_group.len()); - for (labels, value) in answers { - let key = ( - labels["region"].clone(), - labels["service"].clone(), - labels["job"].clone(), - ); - // The exact answer: the mean of the group's per-series p99s, each - // PromQL's interpolated quantile of the series' window. - let exact = per_group[&key] - .iter() - .map(|window| { - let mut sorted = window.clone(); - sorted.sort_by(f64::total_cmp); - let rank = 0.99 * (sorted.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - sorted[low] + (sorted[high] - sorted[low]) * (rank - low as f64) - }) - .sum::() - / per_group[&key].len() as f64; - assert!( - (value - exact).abs() <= 0.05 * exact, - "{labels:?}: {value} vs exact {exact}" - ); - } -} - -/// Without a non-negative declaration, relative sketch error cannot propagate -/// through the average, so this no-raw deployment has no valid plan: plan -/// selection selects none. Pass 1 prunes the quantile sketches for that -/// reason (`NoErrorBoundForQueryTarget`) and Stage 2 the exact candidates, -/// which scan raw data at query time. -#[tokio::test] -async fn an_undeclared_gauges_merged_ddsketch_tumbling_windows_get_no_relative_bound_through_avg() { - let error = plan_p99_avg(false).await.expect_err("no valid plan"); - assert_plan_selection_selected_none(&error); - // Pass 1 prunes both quantile sketches for that reason: DDSketch's - // relative bound does not pass through the average of an undeclared - // gauge, and no rule carries KLL's rank bound through it. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); - let no_bound = pass1_pruned_for_no_error_bound(&p99_avg_workload(false), models); - assert!( - no_bound.contains(&"DDSketch".to_string()) && no_bound.contains(&"Kll".to_string()), - "{no_bound:?}" - ); - // Declared non-negative, DDSketch's bound passes through the average. - let no_bound = pass1_pruned_for_no_error_bound(&p99_avg_workload(true), models); - assert!(!no_bound.contains(&"DDSketch".to_string()), "{no_bound:?}"); -} - -/// The summaries Pass 1 pruned planning `workload` (PromQL) with `models` -/// because no query reading them can meet its target -/// ([`PruneReason::NoErrorBoundForQueryTarget`]), as the facade's planning -/// run reports them (`PlanningRun::per_computation_candidates`, its -/// `pruned_candidates`): each family's name (e.g. `"Kll"`). -fn pass1_pruned_for_no_error_bound( - workload: &PlanningWorkload, - models: PlanningModels<'_>, -) -> Vec { - let roots = asap_frontend_promql::lower_promql_query_workload(workload) - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(with_promql_series_identity(&node).unwrap_or(node)) - } - scalar => scalar, - }) - .collect(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let run = run_planning( - roots, - &entries, - &data, - models, - PlanSearchSettings { inspect: true }, - ) - .unwrap(); - run.per_computation_candidates - .expect("Pass 1's output") - .pruned_candidates - .iter() - .filter(|pruned| pruned.reason == PruneReason::NoErrorBoundForQueryTarget) - .filter_map(|pruned| match &pruned.candidate { - HowToComputeAggregateMeasure::Summary(family) => Some(format!("{family:?}")), - _ => None, - }) - .collect() -} - -/// `error` says plan selection found no valid plan: no candidate is valid. -/// Not a solver, realizer or input failure. -fn assert_plan_selection_selected_none(error: &str) { - assert!( - error.contains("plan selection: no valid candidate"), - "{error}" - ); -} - -/// `avg by (g) (quantile_over_time(0.99, x[100s]))` every 10 s at rank -/// error 0.01 and δ 0.01, over 100 ingesting `x` series sampled every -/// second: ASAPQuery's random mixed workload query. Stage 1 has no -/// analytical rule for an average of quantile estimates, so every sketch -/// candidate has no guarantee. -fn average_of_p99s_workload() -> PlanningWorkload { - serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "avg by (g) (quantile_over_time(0.99, x[100s]))", - "demand": {"fixed_interval": 10000}, - "requirements": {"accuracy": {"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "per_value"}}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 100.0}}, - "sources": {"x": {"series_count": 100, "sample_interval": 1000}}}}"#, - ) - .unwrap() -} - -/// Raw `x` samples of series `a` and `b` in group `g="1"` and `c` in group -/// `g="2"`, one every second from 200 s before `EVALUATION_MS` to it, off -/// the second boundaries: series `i`'s value is `100 × i + 1..=101`, all -/// positive, which DDSketch needs at run time (nothing declares it here). -fn grouped_samples() -> Vec { - let mut samples = Vec::new(); - for (i, (instance, g)) in [("a", "1"), ("b", "1"), ("c", "2")].iter().enumerate() { - for step in 0..200 { - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "x".to_string()), - ("instance".to_string(), instance.to_string()), - ("g".to_string(), g.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 199_500 + step * 1_000, - value: (100 * i) as f64 + 1.0 + ((step * 37 + 11 * i as i64) % 101) as f64, - }); - } - } - samples -} - -/// Quantile queries over one metric and window with different `by` labels -/// (ASAPQuery's seed-862 quantile queries) share one per-series DDSketch: -/// planned jointly on a deployment that keeps no raw data samples, every stored -/// DDSketch has one definition, and each query answers as its query-time -/// reference. Series of `data` carry `region`, `service` and `job`; values -/// are positive. -#[tokio::test] -async fn quantile_queries_with_different_groupings_share_one_per_series_sketch() { - let plan_json = jointly_planned( - &[ - "avg by (region) (quantile_over_time(0.99, data[100s]))", - "avg by (region, service, job) (quantile_over_time(0.9, data[100s]))", - "avg by (service) (quantile_over_time(0.5, data[100s]))", - ], - r#"{"accuracy": {"accuracy_metric": "relative", "max_error": 0.05, - "failure_probability": 0.01, "accuracy_scope": "per_value"}}"#, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let sketches = stored_definitions(&plan, "DDSketch"); - assert_eq!(sketches.len(), 1, "{sketches:?}"); - let run = run_data_plane( - &plan_json, - &grouped_series_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query[0].len(), 2, "one answer per region"); - assert_eq!(run.answers_per_query[1].len(), 6); - assert_eq!(run.answers_per_query[2].len(), 3, "one answer per service"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// The six series of `data` the sharing tests read: `region` east/west, -/// `service` api/web/db, `job` batch/online; values positive, one every -/// 10 s over the last two minutes, in time order. -fn grouped_series_samples() -> Vec { - let mut samples = Vec::new(); - for (instance, region, service, job) in [ - ("s1", "east", "api", "online"), - ("s2", "east", "web", "online"), - ("s3", "east", "db", "batch"), - ("s4", "west", "api", "online"), - ("s5", "west", "web", "batch"), - ("s6", "west", "db", "batch"), - ] { - let base: f64 = instance[1..].parse().unwrap(); - for step in (0..12).rev() { - let mut sample = sample( - "data", - instance, - EVALUATION_MS - 5_000 - step * 10_000, - base * 10.0 + ((step * 37) % 11) as f64, - ); - sample.labels.insert("region".into(), region.into()); - sample.labels.insert("service".into(), service.into()); - sample.labels.insert("job".into(), job.into()); - samples.push(sample); - } - } - samples -} - -/// The exact answer of `avg by (g) (quantile_over_time(0.99, x[100s]))` at -/// `EVALUATION_MS` over [`grouped_samples`]: PromQL's p99 (linear -/// interpolation at rank `0.99 × (n − 1)`) of each series' samples in -/// `(EVALUATION_MS − 100 s, EVALUATION_MS]`, averaged per group. -fn exact_average_of_p99s(samples: &[RawDataSample]) -> BTreeMap { - let mut per_series: BTreeMap<(String, String), Vec> = BTreeMap::new(); - for s in samples - .iter() - .filter(|s| EVALUATION_MS - 100_000 < s.timestamp_ms && s.timestamp_ms <= EVALUATION_MS) - { - per_series - .entry((s.labels["g"].clone(), s.labels["instance"].clone())) - .or_default() - .push(s.value); - } - let mut per_group: BTreeMap> = BTreeMap::new(); - for ((g, _), mut values) in per_series { - values.sort_by(f64::total_cmp); - let rank = 0.99 * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - let p99 = values[low] + (values[high] - values[low]) * (rank - low as f64); - per_group.entry(g).or_default().push(p99); - } - per_group - .into_iter() - .map(|(g, p99s)| (g, p99s.iter().sum::() / p99s.len() as f64)) - .collect() -} - -/// Issue #774: without raw data samples retained, every candidate of an average -/// of p99s fails a check (the exact ones scan raw data samples at query time, -/// the sketch ones have no guarantee), so planning fails. With -/// `PlanningModels::ignore_accuracy_guarantees`, the cheapest -/// sketch plan is selected, the report marks its output as accepted without -/// a guarantee, and it runs: equal to its query-time reference and within a -/// loose tolerance of the exact answer. -#[tokio::test] -async fn ignoring_accuracy_guarantees_selects_a_sketch_plan_that_runs() { - let workload = average_of_p99s_workload(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan = |ignore: bool| { - let workload = workload.clone(); - let capabilities = capabilities.clone(); - async move { - plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_ignore_accuracy_guarantees(ignore), - )) - .await - } - }; - // Without the setting, no candidate is valid. - let error = plan(false) - .await - .expect_err("no candidate is valid without the setting"); - assert_plan_selection_selected_none(&error.to_string()); - // Pass 1 prunes every quantile sketch for it: no rule carries a rank - // bound through the average, and a relative one does not meet a rank - // target. With the setting, Stage 3 may accept an output without its - // guarantee, so Pass 1 prunes none. - let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); - let no_bound = pass1_pruned_for_no_error_bound(&workload, models); - assert!( - no_bound.contains(&"DDSketch".to_string()) && no_bound.contains(&"Kll".to_string()), - "{no_bound:?}" - ); - assert_eq!( - pass1_pruned_for_no_error_bound(&workload, models.with_ignore_accuracy_guarantees(true)), - Vec::::new() - ); - // With it. - let (installed, report) = plan(true).await.expect("a sketch plan is selected"); - // A per-series KLL or DDSketch plan. - let families: Vec = installed - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => Some(format!("{family:?}")), - _ => None, - }) - .collect(); - // Its sketches (beside the average's exact sum and count) are KLL or - // DDSketch. - let sketches: Vec<&String> = families - .iter() - .filter(|f| f.starts_with("Sketch")) - .collect(); - assert!( - !sketches.is_empty() - && sketches - .iter() - .all(|f| f.contains("Kll") || f.contains("DDSketch")), - "{families:?}" - ); - // The report marks the query's output as accepted without a guarantee. - let report = report.as_ref().expect("Stage 3's report"); - assert_eq!(report.queries_accepted_without_guarantee, vec![0]); - // It runs, equal to its query-time reference ... - let plan_json = serde_json::to_string(&installed).unwrap(); - let samples = grouped_samples(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - // ... and within 5 (of values spanning 100 per series) of the exact - // average of p99s, per group. - let exact = exact_average_of_p99s(&samples); - let answered: BTreeMap = run.answers_per_query[0] - .iter() - .map(|(labels, value)| (labels["g"].clone(), *value)) - .collect(); - assert_eq!( - answered.keys().collect::>(), - exact.keys().collect::>() - ); - for (g, value) in &answered { - assert!( - (value - exact[g]).abs() <= 5.0, - "g={g}: {value} vs exact {}", - exact[g] - ); - } -} - -/// The MVP benchmark's p99 query over an undeclared gauge, which has no -/// valid plan with the built-in model, gets one with -/// `ignore_accuracy_guarantees`, and the report marks its output as -/// accepted without a guarantee. -#[tokio::test] -async fn ignoring_accuracy_guarantees_plans_the_undeclared_p99_query() { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let settings = CostInputs::default(); - let (plan, report) = plan_installable(UserInput::new( - &p99_avg_workload(false), - FrontendInput::Promql, - PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(true), - )) - .await - .expect("a plan is selected"); - let report = report.as_ref().expect("Stage 3's report"); - assert_eq!(report.queries_accepted_without_guarantee, vec![0]); - assert!( - plan.raw_data_retention().unwrap().is_empty(), - "no raw samples read" - ); -} - -/// [`p99_avg_workload`]'s query with `outer` in place of `avg`, planned -/// for a deployment that keeps no raw data samples. -async fn plan_p99( - outer: &str, - values_non_negative: bool, - ignore_accuracy: bool, -) -> (InstallablePlan, Option) { - let mut json = serde_json::to_value(p99_avg_workload(values_non_negative)).unwrap(); - json["query_workload"]["repeating_queries"][0]["query"] = - format!("{outer} by (region, service, job) (quantile_over_time(0.99, data[100s]))").into(); - let workload: PlanningWorkload = serde_json::from_value(json).unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - // A DDSketch kernel fixture, not a comparison of sketch families. - supported_summaries: ignore_accuracy - .then(|| only_summary_family(SketchAlgorithm::DDSketch)), - ..DeploymentCapabilities::default() - }; - let settings = CostInputs::default(); - let models = PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(ignore_accuracy); - plan_installable(UserInput::new(&workload, FrontendInput::Promql, models)) - .await - .expect("a valid plan") -} - -/// `data` samples of series `0..series`, `value(series, step)` every -/// `interval_ms` over the `steps` steps ending just before `EVALUATION_MS`, -/// two series per (region, service, job) group; and each group's series' -/// windows `(EVALUATION_MS - 100 s, EVALUATION_MS]`. -#[allow(clippy::type_complexity)] -fn data_samples( - series: usize, - steps: i64, - interval_ms: i64, - value: impl Fn(usize, i64) -> f64, -) -> ( - Vec, - BTreeMap<(String, String, String), Vec>>, -) { - let mut samples = Vec::new(); - let mut per_group: BTreeMap<(String, String, String), Vec>> = BTreeMap::new(); - for s in 0..series { - let (region, service, job) = ( - format!("r{}", s % 2), - format!("s{}", s / 4), - "j".to_string(), - ); - let mut window = Vec::new(); - for step in 0..steps { - let timestamp_ms = EVALUATION_MS - steps * interval_ms + 50 + step * interval_ms; - let v = value(s, step); - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "data".to_string()), - ("region".to_string(), region.clone()), - ("service".to_string(), service.clone()), - ("job".to_string(), job.clone()), - ("instance".to_string(), format!("i{s}")), - ]), - timestamp_ms, - value: v, - }); - if timestamp_ms > EVALUATION_MS - 100_000 { - window.push(v); - } - } - per_group - .entry((region, service, job)) - .or_default() - .push(window); - } - (samples, per_group) -} - -/// PromQL's interpolated `quantile_over_time(q, ...)` of `window`. -fn interpolated_quantile(q: f64, window: &[f64]) -> f64 { - let mut sorted = window.to_vec(); - sorted.sort_by(f64::total_cmp); - let rank = q * (sorted.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - sorted[low] + (sorted[high] - sorted[low]) * (rank - low as f64) -} - -/// Runs `plan` over `samples` and checks every group's answer against -/// `combine` of its series' exact p99s, within the relative error 0.05, and -/// against the query-time reference. -fn assert_p99s_within_alpha( - plan: &InstallablePlan, - samples: &[RawDataSample], - per_group: &BTreeMap<(String, String, String), Vec>>, - combine: impl Fn(&[f64]) -> f64, -) { - let run = run_data_plane( - &serde_json::to_string(plan).unwrap(), - samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - let answers = &run.answers_per_query[0]; - assert_eq!(answers.len(), per_group.len()); - for (labels, value) in answers { - let key = ( - labels["region"].clone(), - labels["service"].clone(), - labels["job"].clone(), - ); - let p99s: Vec = per_group[&key] - .iter() - .map(|window| interpolated_quantile(0.99, window)) - .collect(); - let exact = combine(&p99s); - assert!( - (value - exact).abs() <= 0.05 * exact.abs(), - "{labels:?}: {value} vs exact {exact}" - ); - } -} - -/// A non-negative gauge that reports exactly 0 is still summarized by the -/// selected DDSketch tumbling windows (zero has its own bucket): each series' window -/// holds the samples 0, 0, 3 and 5 scaled by the series, and every answer -/// equals the query-time reference's and is within the relative error 0.05 -/// of the exact average of PromQL's interpolated p99s. -#[tokio::test] -async fn a_ddsketch_over_a_gauge_reporting_zero_runs_within_alpha() { - let (plan, _) = plan_p99("avg", true, false).await; - let views = plan.materialized_views().unwrap(); - assert!(!views.is_empty(), "tumbling windows built at ingestion"); - let (samples, per_group) = data_samples(8, 4, 20_000, |s, step| { - (s + 1) as f64 * [0.0, 0.0, 3.0, 5.0][step as usize] - }); - assert!(per_group.values().flatten().all(|window| window.len() == 4)); - let average = |p99s: &[f64]| p99s.iter().sum::() / p99s.len() as f64; - assert_p99s_within_alpha(&plan, &samples, &per_group, average); - // Many samples, a third of them zero. - let (samples, per_group) = data_samples(8, 1_100, 100, |s, step| { - (s + 1) as f64 * [0.0, 3.0, 5.0][(step % 3) as usize] - }); - assert_p99s_within_alpha(&plan, &samples, &per_group, average); -} - -/// Signed samples (negative, zero and positive) are summarized by the -/// selected DDSketch tumbling windows, through their negative store and zero bucket: -/// Signed interpolation has no general relative guarantee, so this kernel -/// fixture explicitly bypasses accuracy admission. Every answer equals the -/// query-time reference's and is -/// within 0.05 of the exact value, including series whose samples are all -/// negative. -#[tokio::test] -async fn a_ddsketch_over_signed_samples_runs_within_alpha() { - let (plan, report) = plan_p99("max", false, true).await; - let views = plan.materialized_views().unwrap(); - assert!( - !stored_definitions(&plan, "DDSketch").is_empty(), - "exercise signed DDS state" - ); - assert_eq!( - report.as_ref().unwrap().queries_accepted_without_guarantee, - vec![0] - ); - assert!(!views.is_empty(), "tumbling windows built at ingestion"); - let max = |p99s: &[f64]| p99s.iter().copied().fold(f64::NEG_INFINITY, f64::max); - // Series 0..4 cross zero; series 4..8 are all negative. - let (samples, per_group) = data_samples(8, 1_100, 100, |s, step| { - let scale = (s + 1) as f64; - if s < 4 { - scale * [-7.0, -3.0, 0.0, 2.0, 4.0][(step % 5) as usize] - } else { - scale * [-9.0, -8.0, -5.0, -1.0][(step % 4) as usize] - } - }); - assert_p99s_within_alpha(&plan, &samples, &per_group, max); } /// The ASAPQuery workloads' counter top-k query `topk(10, sum by (job) @@ -2750,111 +388,6 @@ fn counter_topk_workload(function: &str) -> PlanningWorkload { .unwrap() } -/// The per-second rate of series `instance` (1 or 2) of job `job` (1..=12) -/// in [`counter_raw_data_samples`]: distinct for every series and every job's sum. -fn counter_rate(job: i64, instance: i64) -> f64 { - job as f64 + 0.1 * instance as f64 -} - -/// `data_total` samples of 12 jobs with 2 series each, every 10 s from -/// 595 s before `EVALUATION_MS`: each a counter growing linearly at -/// [`counter_rate`], so its 5-minute `increase` is 300 × its rate (the -/// samples 5 s inside each window end extrapolate to them exactly). -fn counter_raw_data_samples() -> Vec { - let mut samples = Vec::new(); - for job in 1..=12 { - for instance in 1..=2 { - for step in 0..60 { - let mut s = sample( - "data_total", - &format!("{job}-{instance}"), - EVALUATION_MS - 595_000 + step * 10_000, - counter_rate(job, instance) * (5 + step * 10) as f64, - ); - s.labels.insert("job".into(), format!("j{job:02}")); - samples.push(s); - } - } - } - samples -} - -/// Issue: with guarantees ignored and no raw data samples, the counter top-k -/// queries selected exact per-series tumbling windows followed by a query-time -/// CountSketchWithHeap top-k the executor cannot compile ("invalid keyed -/// evaluation shape or capacity"). The selected plan must be one the -/// executor compiles; it runs, equal to its query-time reference and to the -/// exact top 10 jobs (jobs 3..=12, each the sum of its two series). -#[tokio::test] -async fn ignoring_guarantees_counter_topk_plans_compile_and_answer_exactly() { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let settings = CostInputs::default(); - for (function, seconds) in [("increase", 300.0), ("rate", 1.0)] { - let plan = plan_installable(UserInput::new( - &counter_topk_workload(function), - FrontendInput::Promql, - PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(true), - )) - .await - .unwrap_or_else(|error| panic!("{function}: {error}")) - .0; - // An exact plan: no heap over the per-job sums, which have one row - // per job. - assert!( - !plan.dag.nodes.iter().any(|node| matches!( - &node.payload, - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { - family: asap_executor::planner::ir::schema::FieldDataType::Sketch(..), - .. - } - ) - )), - "{function}" - ); - // The plan compiles as a runtime compiles it. - plan.check_compiles().unwrap(); - let plan_json = serde_json::to_string(&plan).unwrap(); - let run = run_data_plane( - &plan_json, - &counter_raw_data_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!( - run.answers_per_query, run.reference_answers_per_query, - "{function}" - ); - let expected: Vec<(String, f64)> = (3..=12) - .map(|job| { - ( - format!("j{job:02}"), - (counter_rate(job, 1) + counter_rate(job, 2)) * seconds, - ) - }) - .collect(); - let mut answered: Vec<(String, f64)> = run.answers_per_query[0] - .iter() - .map(|(labels, value)| (labels["job"].clone(), *value)) - .collect(); - answered.sort_by(|a, b| a.0.cmp(&b.0)); - assert_eq!(answered.len(), expected.len(), "{function}: {answered:?}"); - for ((job, value), (want_job, want)) in answered.iter().zip(&expected) { - assert_eq!(job, want_job, "{function}"); - assert!( - (value - want).abs() <= 1e-9 * want, - "{function} {job}: {value} vs {want}" - ); - } - } -} - /// `InstallablePlan::check_compiles` reports what the Planner's executor /// refuses: the counter top-k plan with its answer column's type changed /// from Float64 to Utf8 does not compile, with the executor's reason. @@ -2895,897 +428,6 @@ async fn check_compiles_reports_the_executors_error() { ); } -/// `queries` every 10 s at `accuracy` (JSON) over six ingesting series of -/// `data`, a non-negative gauge, planned jointly on a deployment that keeps -/// no raw data samples: the plan file. -async fn jointly_planned(queries: &[&str], accuracy: &str) -> String { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {accuracy}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 0.6}}}}, - "sources": {{"data": {{"series_count": 6, "sample_interval": 10000, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, - entries.join(",") - )) - .unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - plan_json_for(workload, &capabilities).await -} - -/// The definition ids of `plan`'s materialized views whose node's payload -/// mentions `family` (e.g. `"DDSketch"`, `"Sum"`). -fn stored_definitions(plan: &InstallablePlan, family: &str) -> BTreeSet { - plan.materialized_views() - .unwrap() - .iter() - .filter(|view| { - let node = plan - .dag - .nodes - .iter() - .find(|n| n.id == view.node_id) - .unwrap(); - serde_json::to_string(&node.payload) - .unwrap() - .contains(family) - }) - .map(|view| view.coverage().definition_id().to_owned()) - .collect() -} - -/// A fine and a coarse sum over one metric and window (ASAPQuery's -/// sum-fine and sum-coarse) store one Sum: both group sums are computed -/// from one stored per-series Sum (Stage 1 shares the per-series sums the -/// two queries' scans read), so every stored Sum has one definition, and -/// both answer as the query-time reference. -#[tokio::test] -async fn fine_and_coarse_sums_store_one_sum() { - let plan_json = jointly_planned( - &[ - "sum by (region, service, job) (sum_over_time(data[100s]))", - "sum by (region) (sum_over_time(data[100s]))", - ], - "{}", - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let sums = stored_definitions(&plan, "\"Sum\""); - assert_eq!(sums.len(), 1, "{sums:?}"); - let run = run_data_plane( - &plan_json, - &grouped_series_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query[0].len(), 6); - assert_eq!(run.answers_per_query[1].len(), 2, "one sum per region"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Per-series answers planned beside a grouping of the same per-series -/// values (over a scan with other labels) answer as their query-time -/// reference, and the two share one stored per-series state: a top-k of -/// per-series sums keeps each chosen series' labels, and per-series -/// DDSketch quantiles keep every series' labels. -#[tokio::test] -async fn per_series_answers_beside_a_grouping_share_one_state_and_answer_as_the_reference() { - for (queries, family, accuracy, answers) in [ - ( - [ - "topk(2, sum_over_time(data[100s]))", - "sum by (region) (sum_over_time(data[100s]))", - ], - "\"Sum\"", - "{}", - 2, - ), - ( - [ - "quantile_over_time(0.9, data[100s])", - "avg by (region) (quantile_over_time(0.99, data[100s]))", - ], - "DDSketch", - r#"{"accuracy": {"accuracy_metric": "relative", "max_error": 0.05, - "failure_probability": 0.01, "accuracy_scope": "per_value"}}"#, - 6, - ), - ] { - let plan_json = jointly_planned(&queries, accuracy).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert_eq!(stored_definitions(&plan, family).len(), 1, "{queries:?}"); - let run = run_data_plane( - &plan_json, - &grouped_series_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query[0].len(), answers, "{queries:?}"); - assert_eq!(run.answers_per_query[1].len(), 2, "{queries:?}: per region"); - assert_eq!( - run.answers_per_query, run.reference_answers_per_query, - "{queries:?}" - ); - } -} - -/// A heap top-k query `query` (the ASAPQuery workloads' top-k-sum or -/// counter top-k shape) every 10 s, within 0.05 at failure probability 0.3, -/// over 10,000 series of `metric` sampled every 100 ms, with `facts` (its -/// type and sign, as JSON fields). -fn heap_topk_workload(query: &str, metric: &str, facts: &str) -> PlanningWorkload { - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ - "query": {query:?}, - "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 100000.0}}}}, - "sources": {{"{metric}": {{"series_count": 10000, "sample_interval": 100, - {facts}}}}}}}}}"# - )) - .unwrap() -} - -/// The executor's support for `family` alone: a deployment whose only -/// sketch family is `family` (exact plans stay allowed). -fn only_summary_family(family: SketchAlgorithm) -> Vec { - asap_executor::supported_summary_families() - .into_iter() - .filter(|supported| supported.family == family) - .collect() -} - -/// A deployment's accuracy model that accepts, whatever its guarantee, only -/// a query output that reads a heap of this algorithm, and no other output: -/// it forces the cheapest plan with that heap, as a deployment that requires -/// the heap would. -struct HeapPlansOnly(SketchAlgorithm); - -impl asap_accuracy::AccuracyModel for HeapPlansOnly { - fn satisfies( - &self, - _guarantee: &asap_types::ir::properties::AccuracyGuarantee, - _target: &asap_types::ir::properties::AccuracyTarget, - ) -> bool { - false - } - - fn accepts_output_regardless_of_guarantee( - &self, - output: &QueryRoot, - _target: &asap_types::ir::properties::AccuracyTarget, - ) -> bool { - let QueryRoot::Operator(root) = output else { - return false; - }; - OperatorNode::reachable(root).iter().any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. } - | ASAPOp::SummaryTumblingWindowChain { family: FieldDataType::Sketch(kind, _), .. }) - if *kind.algorithm() == self.0) - }) - } -} - -/// `workload` planned with accuracy guarantees ignored (a top-k estimate -/// has none) on a deployment that keeps no raw samples; with `heap`, the -/// deployment's accuracy model accepts only outputs that read a heap of that -/// algorithm ([`HeapPlansOnly`]). -async fn plan_ignoring_guarantees( - workload: &PlanningWorkload, - heap: Option, -) -> Result { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - // Ignoring guarantees replaces the accuracy model, so a forced heap goes - // through its own accuracy model instead. - let settings = CostInputs::default(); - let heap_only = heap.map(HeapPlansOnly); - let models = PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(heap.is_none()); - let models = match &heap_only { - Some(model) => models.with_accuracy(model), - None => models, - }; - plan_installable(UserInput::new(workload, FrontendInput::Promql, models)) - .await - .map(|(plan, _)| plan) - .map_err(|e| e.to_string()) -} - -/// The heap sketch `plan` builds: its algorithm and width. -fn heap_of(plan: &InstallablePlan) -> Option<(SketchAlgorithm, u32)> { - plan.dag.nodes.iter().find_map(|node| match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => match family { - FieldDataType::Sketch(kind, _) => match kind.params() { - SketchParams::CmsWithHeap { width, .. } - | SketchParams::CountSketchWithHeap { width, .. } => { - Some((*kind.algorithm(), *width)) - } - _ => None, - }, - _ => None, - }, - _ => None, - }) -} - -/// The heap of `algorithm`'s point-error bound on an item of a partition -/// whose items' true values are `values`: `ε·L1` for Count-Min (ε = e/w, -/// an overestimate), `ε·L2` for Count Sketch (ε = √(3/w)), which is at most -/// `ε·L1`. -fn heap_bound(algorithm: &SketchAlgorithm, width: u32, values: &[f64]) -> f64 { - let width = f64::from(width); - match algorithm { - SketchAlgorithm::CmsWithHeap => { - std::f64::consts::E / width * values.iter().map(|v| v.abs()).sum::() - } - SketchAlgorithm::CountSketchWithHeap => { - (3.0 / width).sqrt() * values.iter().map(|v| v * v).sum::().sqrt() - } - other => panic!("not a heap: {other:?}"), - } -} - -/// Checks a heap top-k plan's `answers` against the `truth` (each item's -/// exact value, by partition): per partition, `k` answers, the true top `k` -/// recalled, each item named by its labels (`item_label`, and `partition` -/// when partitioned), and each estimate within the heap's bound. -fn assert_heap_topk_answers( - answers: &[Answer], - truth: &BTreeMap>, - (partition, item_label, k): (Option<&str>, &str, usize), - (algorithm, width): (&SketchAlgorithm, u32), -) { - let mut answered: BTreeMap> = BTreeMap::new(); - for (labels, value) in answers { - // A range function's result drops the metric name, as the exact - // top-k's does. - assert!(!labels.contains_key("__name__"), "{labels:?}"); - let group = partition.map_or(String::new(), |p| labels[p].clone()); - let item = labels - .get(item_label) - .unwrap_or_else(|| panic!("{item_label} kept: {labels:?}")); - answered - .entry(group) - .or_default() - .push((item.clone(), *value)); - } - assert_eq!( - answered.keys().collect::>(), - truth.keys().collect::>() - ); - for (group, items) in &answered { - let exact = &truth[group]; - let bound = heap_bound( - algorithm, - width, - &exact.values().copied().collect::>(), - ); - let mut ranked: Vec<(&String, &f64)> = exact.iter().collect(); - ranked.sort_by(|a, b| b.1.total_cmp(a.1)); - let top: BTreeSet<&String> = ranked.iter().take(k).map(|(item, _)| *item).collect(); - assert_eq!(items.len(), k, "{group}: {items:?}"); - let recalled = items.iter().filter(|(item, _)| top.contains(item)).count(); - assert_eq!(recalled, k, "{group}: recall of {top:?} in {items:?}"); - for (item, estimate) in items { - let error = estimate - exact[item]; - assert!( - error.abs() <= bound + 1e-9 * exact[item].abs(), - "{group}/{item}: {estimate} vs exact {}, bound {bound}", - exact[item] - ); - if *algorithm == SketchAlgorithm::CmsWithHeap { - assert!( - error >= -1e-9 * exact[item].abs(), - "Count-Min overestimates" - ); - } - } - } -} - -/// `data` samples of 2 regions with 40 series each, six inside the last -/// minute per series: series `r` (rank 1..=40) of a region sums to -/// `1000 / r^1.1` (west at half that), a skewed ranking. -fn skewed_series_samples() -> (Vec, BTreeMap>) { - let mut samples = Vec::new(); - let mut truth: BTreeMap> = BTreeMap::new(); - for (region, scale) in [("east", 1.0), ("west", 0.5)] { - for rank in 1..=40 { - let instance = format!("{region}-{rank:02}"); - let sum = scale * 1000.0 / f64::from(rank).powf(1.1); - for step in 0..6 { - let mut s = sample( - "data", - &instance, - EVALUATION_MS - 55_000 + step * 10_000, - sum / 6.0, - ); - s.labels.insert("region".into(), region.into()); - samples.push(s); - } - truth - .entry(region.to_string()) - .or_default() - .insert(instance, sum); - } - } - (samples, truth) -} - -/// `workload`'s plan with only `algorithm`'s heap plans accepted -/// ([`HeapPlansOnly`] forces the heap; guarantees are not ignored): it builds that heap, compiles as a runtime compiles it, and -/// both it and its query-time reference answer `samples` with the true top -/// `k` per partition, each named by its labels and within the heap's bound -/// of `truth` ([`assert_heap_topk_answers`]). -async fn assert_forced_heap_answers( - workload: &PlanningWorkload, - algorithm: SketchAlgorithm, - (samples, truth): (&[RawDataSample], &BTreeMap>), - labels: (Option<&str>, &str, usize), -) { - let plan = plan_ignoring_guarantees(workload, Some(algorithm)) - .await - .unwrap_or_else(|error| panic!("{algorithm:?}: {error}")); - let (built, width) = heap_of(&plan).expect("a heap plan"); - assert_eq!(built, algorithm); - plan.check_compiles().unwrap(); - let plan_json = serde_json::to_string(&plan).unwrap(); - let run = run_data_plane(&plan_json, samples, EVALUATION_MS, &mut SummaryStore::new()); - for answers in [ - &run.answers_per_query[0], - &run.reference_answers_per_query[0], - ] { - assert_heap_topk_answers(answers, truth, labels, (&algorithm, width)); - } -} - -/// The ASAPQuery workloads' top-k-sum shape over a gauge, -/// `topk by (region) (3, sum_over_time(data[1m]))`: the fused -/// whole-expression Count Sketch heap (one per region over the raw -/// samples, ranking series identities), forced with guarantees ignored, -/// compiles, and answers each region's true top 3 series, with their full -/// labels, within its bound of their exact sums. (A Count-Min heap needs -/// the weights proven non-negative.) -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_topk_sum_plan_compiles_and_answers_within_its_bound() { - let workload = heap_topk_workload( - "topk by (region) (3, sum_over_time(data[1m]))", - "data", - r#""metric_type": "gauge""#, - ); - let (samples, truth) = skewed_series_samples(); - assert_forced_heap_answers( - &workload, - SketchAlgorithm::CountSketchWithHeap, - (&samples, &truth), - (Some("region"), "instance", 3), - ) - .await; -} - -/// `data` samples of 2 regions with 40 series each inside the last minute: -/// series `r` (rank 1..=40) has `round(200 / r^1.1)` samples (at least 1), -/// a skewed count ranking. -fn skewed_count_samples() -> (Vec, BTreeMap>) { - let mut samples = Vec::new(); - let mut truth: BTreeMap> = BTreeMap::new(); - for region in ["east", "west"] { - for rank in 1..=40 { - let instance = format!("{region}-{rank:02}"); - let count = (200.0 / f64::from(rank).powf(1.1)).round().max(1.0) as i64; - for step in 0..count { - // Spread over (EVALUATION_MS - 59 s, EVALUATION_MS - 1 s]. - let at = EVALUATION_MS - 1_000 - step * 58_000 / count; - let mut s = sample("data", &instance, at, 1.0 + step as f64); - s.labels.insert("region".into(), region.into()); - samples.push(s); - } - truth - .entry(region.to_string()) - .or_default() - .insert(instance, count as f64); - } - } - samples.sort_by_key(|s| s.timestamp_ms); - (samples, truth) -} - -/// `topk by (region) (3, count_over_time(data[1m]))`: each forced fused -/// heap (each sample counts 1) compiles and answers each region's true top -/// 3 series, with their labels, within its bound of their counts. -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_topk_count_plan_compiles_and_answers_within_its_bound() { - let workload = heap_topk_workload( - "topk by (region) (3, count_over_time(data[1m]))", - "data", - r#""metric_type": "gauge""#, - ); - let (samples, truth) = skewed_count_samples(); - for algorithm in [ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ] { - assert_forced_heap_answers( - &workload, - algorithm, - (&samples, &truth), - (Some("region"), "instance", 3), - ) - .await; - } -} - -/// The per-second rate of series `instance` (1 or 2) of job `job` (1..=12) -/// in [`skewed_counter_raw_data_samples`]: `50 / job^1.2`, plus a little per -/// instance, a skewed ranking of the jobs' sums. -fn skewed_counter_rate(job: i64, instance: i64) -> f64 { - 50.0 / (job as f64).powf(1.2) + 0.01 * instance as f64 -} - -/// `data_total` samples of 12 jobs with 2 series each, every 10 s from 595 s -/// before `EVALUATION_MS`, each a counter growing linearly at -/// [`skewed_counter_rate`], as [`counter_raw_data_samples`]. -fn skewed_counter_raw_data_samples() -> Vec { - let mut samples = Vec::new(); - for job in 1..=12 { - for instance in 1..=2 { - for step in 0..60 { - let mut s = sample( - "data_total", - &format!("{job}-{instance}"), - EVALUATION_MS - 595_000 + step * 10_000, - skewed_counter_rate(job, instance) * (5 + step * 10) as f64, - ); - s.labels.insert("job".into(), format!("j{job:02}")); - samples.push(s); - } - } - } - samples -} - -/// The ASAPQuery workloads' counter top-k, `topk(3, sum by (job) -/// ((data_total[5m])))` for `f` `increase` or `rate`: each forced fused -/// heap (one over the per-series values, ranking jobs) compiles and answers -/// the true top 3 jobs, with their `job` labels, within its bound of their -/// exact sums. -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_counter_topk_plan_compiles_and_answers_within_its_bound() { - let samples = skewed_counter_raw_data_samples(); - for (function, seconds) in [("increase", 300.0), ("rate", 1.0)] { - let workload = heap_topk_workload( - &format!("topk(3, sum by (job) ({function}(data_total[5m])))"), - "data_total", - r#""metric_type": "counter""#, - ); - // Each job's exact sum over its two series. - let truth = BTreeMap::from([( - String::new(), - (1..=12) - .map(|job| { - let sum = skewed_counter_rate(job, 1) + skewed_counter_rate(job, 2); - (format!("j{job:02}"), sum * seconds) - }) - .collect::>(), - )]); - for algorithm in [ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ] { - assert_forced_heap_answers(&workload, algorithm, (&samples, &truth), (None, "job", 3)) - .await; - } - } -} - -/// `data` samples of 40 regions with 3 series each, inside the last -/// minute: region `r` (rank 1..=40) sums to `1000 / r^1.1` over -/// its series (a skewed ranking of the regions) and holds `3 · -/// round(200 / r^1.1)` samples (a skewed count ranking), returned with each -/// region's exact sum and exact count. -fn skewed_region_raw_data_samples() -> ( - Vec, - BTreeMap, - BTreeMap, -) { - let mut samples = Vec::new(); - let mut sums = BTreeMap::new(); - let mut counts = BTreeMap::new(); - for rank in 1..=40i64 { - let region = format!("r{rank:02}"); - let sum = 1000.0 / (rank as f64).powf(1.1); - let per_series_steps = (200.0 / (rank as f64).powf(1.1)).round().max(1.0) as i64; - for series in 0..3 { - let instance = format!("{region}-{series}"); - for step in 0..per_series_steps { - // Spread over (EVALUATION_MS - 59 s, EVALUATION_MS - 1 s]. - let at = EVALUATION_MS - 1_000 - step * 58_000 / per_series_steps; - let mut s = sample("data", &instance, at, sum / 3.0 / per_series_steps as f64); - s.labels.insert("region".into(), region.clone()); - samples.push(s); - } - } - sums.insert(region.clone(), sum); - counts.insert(region, (3 * per_series_steps) as f64); - } - samples.sort_by_key(|s| s.timestamp_ms); - (samples, sums, counts) -} - -/// The ASAPQuery workloads' grouped top-k-sum shape, -/// `topk(3, sum by (region) ((data[1m])))` for `f` `sum_over_time` or -/// `count_over_time`: each forced fused heap (one over the raw data samples, -/// keyed by `region`, weighted by the sample value or by one) compiles and -/// answers the true top 3 regions within its bound of their exact sums or -/// counts, each named by its `region` label alone, as the exact top-k's. -/// A Count-Min heap over sample values needs the gauge declared -/// non-negative; a Count Sketch heap takes any gauge. -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_grouped_topk_sum_plan_compiles_and_answers_within_its_bound() { - let (samples, sums, counts) = skewed_region_raw_data_samples(); - let gauge = r#""metric_type": "gauge""#; - let non_negative = r#""metric_type": "gauge", "values_non_negative": true"#; - for (function, truth, facts, algorithm) in [ - ( - "sum_over_time", - &sums, - non_negative, - SketchAlgorithm::CmsWithHeap, - ), - ( - "sum_over_time", - &sums, - gauge, - SketchAlgorithm::CountSketchWithHeap, - ), - ( - "count_over_time", - &counts, - gauge, - SketchAlgorithm::CmsWithHeap, - ), - ( - "count_over_time", - &counts, - gauge, - SketchAlgorithm::CountSketchWithHeap, - ), - ] { - let query = format!("topk(3, sum by (region) ({function}(data[1m])))"); - let workload = heap_topk_workload(&query, "data", facts); - let truth = BTreeMap::from([(String::new(), truth.clone())]); - let plan = plan_ignoring_guarantees(&workload, Some(algorithm)) - .await - .unwrap_or_else(|error| panic!("{query} {algorithm:?}: {error}")); - let (built, width) = heap_of(&plan).expect("a heap plan"); - assert_eq!(built, algorithm, "{query}"); - plan.check_compiles().unwrap(); - let plan_json = serde_json::to_string(&plan).unwrap(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - for answers in [ - &run.answers_per_query[0], - &run.reference_answers_per_query[0], - ] { - assert_heap_topk_answers(answers, &truth, (None, "region", 3), (&algorithm, width)); - for (labels, _) in answers.iter() { - assert_eq!( - labels.keys().collect::>(), - ["region"], - "{query} {algorithm:?}: the group label alone" - ); - } - } - // The installed plan (stored states) answers as its query-time - // reference does, within the heap's bound. - let bound = heap_bound( - &algorithm, - width, - &truth[""].values().copied().collect::>(), - ); - let reference: BTreeMap<_, _> = run.reference_answers_per_query[0] - .iter() - .map(|(labels, value)| (labels.clone(), *value)) - .collect(); - for (labels, value) in &run.answers_per_query[0] { - let expected = reference - .get(labels) - .unwrap_or_else(|| panic!("{query} {algorithm:?}: {labels:?} in the reference")); - assert!( - (value - expected).abs() <= bound, - "{query} {algorithm:?}: {value} vs {expected}" - ); - } - } -} - // ── Classic-histogram `histogram_quantile` (issue #890) ───────────────────── -/// The classic bucket metric of the histogram tests. -const BUCKET_METRIC: &str = "http_request_duration_seconds_bucket"; - -/// Exact PromQL `queries`, every minute, over ingesting classic-bucket -/// series of [`BUCKET_METRIC`] sampled every 10 s. -fn exact_bucket_workload(queries: &[&str]) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 60000}}, - "requirements": {{"accuracy": {{"max_error": 0.0, "failure_probability": 0.0}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 2.4}}}}, - "sources": {{"{BUCKET_METRIC}": {{"series_count": 24, "sample_interval": 10000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// Cumulative bucket counters of [`BUCKET_METRIC`], one sample every 10 s -/// over the 5 minutes before `EVALUATION_MS` (first 295 s before it, last -/// 5 s before it), each rising at a constant per-second rate from 1000, so -/// `rate(…[5m])` of each is its slope. Per `(service, job)`, the slopes of -/// buckets `le` 0.1, 0.5, 1 and `+Inf`, each on instances `1` and `2`: -/// - `orders`/`api`: 0.5, 1.5, 2, 2 (summed over instances: 1, 3, 4, 4); -/// - `batch`/`worker`: 0.5, 0.5, 0.5, 2.5 (most observations above 1, in -/// `+Inf` alone; summed: 1, 1, 1, 5); -/// - `idle`/`idle`: 0 everywhere (no observations). -fn bucket_samples() -> Vec { - let mut samples = Vec::new(); - for (service, job, slopes) in [ - ("orders", "api", [0.5, 1.5, 2.0, 2.0]), - ("batch", "worker", [0.5, 0.5, 0.5, 2.5]), - ("idle", "idle", [0.0; 4]), - ] { - for (le, slope) in ["0.1", "0.5", "1", "+Inf"].into_iter().zip(slopes) { - for instance in ["1", "2"] { - for step in 0..30 { - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), BUCKET_METRIC.to_string()), - ("service".to_string(), service.to_string()), - ("job".to_string(), job.to_string()), - ("instance".to_string(), instance.to_string()), - ("le".to_string(), le.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 295_000 + step * 10_000, - value: 1000.0 + slope * 10.0 * step as f64, - }); - } - } - } - } - samples -} - -/// The plan file of exact `queries` over [`bucket_samples`]' metric, on a -/// deployment that keeps raw data samples. -async fn bucket_plan_json(queries: &[&str]) -> String { - plan_json_for( - exact_bucket_workload(queries), - &DeploymentCapabilities::default(), - ) - .await -} - -/// `histogram_quantile` over classic buckets as the issue's dashboard -/// writes it: p50, p95 and p99 of the summed bucket rates of one service, -/// in one workload. Stage 3 selects an exact plan the executor compiles, -/// and it answers with Prometheus' cumulative-bucket interpolation: over -/// summed rates 1, 3, 4, 4 at `le` 0.1, 0.5, 1, `+Inf`, rank 0.5 · 4 = 2 -/// lies in the 0.5 bucket, 0.1 + 0.4 · (2 − 1) / (3 − 1) = 0.3; rank 3.8 -/// and 3.96 in the 1 bucket, 0.5 + 0.5 · 0.8 = 0.9 and 0.5 + 0.5 · 0.96 = -/// 0.98. The result has no labels: `sum by (le)` keeps only `le`, which -/// the quantile drops. -#[tokio::test] -async fn classic_histogram_quantiles_run_with_prometheus_interpolation() { - let selector = format!(r#"rate({BUCKET_METRIC}{{service="orders"}}[5m])"#); - let queries: Vec = [0.5, 0.95, 0.99] - .iter() - .map(|q| format!("histogram_quantile({q}, sum by (le) ({selector}))")) - .collect(); - let queries: Vec<&str> = queries.iter().map(String::as_str).collect(); - let plan_json = bucket_plan_json(&queries).await; - let run = run_data_plane( - &plan_json, - &bucket_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query.len(), 3); - for (answers, expected) in run.answers_per_query.iter().zip([0.3, 0.9, 0.98]) { - assert_eq!(answers.len(), 1, "{answers:?}"); - let (labels, value) = &answers[0]; - assert!(labels.is_empty(), "le is dropped: {labels:?}"); - assert!((value - expected).abs() < 1e-9, "{value} vs {expected}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// One p99 per `job` (`sum by (job, le)`): the result keeps `job` and -/// drops `le`. `worker`'s rank 0.99 · 5 = 4.95 lies past every finite -/// bucket (most observations are in `+Inf` alone), so Prometheus answers -/// the largest finite bound, 1; `idle` has no observations, so NaN. The -/// p99 alert (`> 0.5`) keeps `api` (0.98) and `worker` (1) and drops -/// `idle` (NaN compares false). A selector that matches no series gives an -/// empty result. Over one series' buckets, without `sum`, the result keeps -/// that series' labels but `le` (and the name `rate` dropped). -#[tokio::test] -async fn a_classic_histogram_quantile_keeps_labels_and_handles_inf_and_empty_input() { - let by_job = format!("histogram_quantile(0.99, sum by (job, le) (rate({BUCKET_METRIC}[5m])))"); - let alert = format!("{by_job} > 0.5"); - let empty = format!( - r#"histogram_quantile(0.5, sum by (le) (rate({BUCKET_METRIC}{{service="missing"}}[5m])))"# - ); - // Without `sum`, each series is its own histogram: `instance` 1's - // buckets alone (rates 0.5, 1.5, 2, 2), keeping its labels but `le`. - let per_series = format!( - r#"histogram_quantile(0.99, rate({BUCKET_METRIC}{{service="orders",instance="1"}}[5m]))"# - ); - let plan_json = bucket_plan_json(&[&by_job, &alert, &empty, &per_series]).await; - let run = run_data_plane( - &plan_json, - &bucket_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let job = |answers: &[Answer]| -> Vec<(String, f64)> { - answers - .iter() - .map(|(labels, value)| { - assert_eq!(labels.keys().collect::>(), ["job"], "{labels:?}"); - (labels["job"].clone(), *value) - }) - .collect() - }; - for answers in [&run.answers_per_query, &run.reference_answers_per_query] { - let per_job = job(&answers[0]); - assert_eq!(per_job.len(), 3, "{per_job:?}"); - assert_eq!(per_job[0].0, "api"); - assert!((per_job[0].1 - 0.98).abs() < 1e-9, "{per_job:?}"); - assert_eq!(per_job[1].0, "idle"); - assert!(per_job[1].1.is_nan(), "{per_job:?}"); - assert_eq!(per_job[2], ("worker".to_string(), 1.0)); - let alerting = job(&answers[1]); - assert_eq!(alerting.len(), 2, "{alerting:?}"); - assert_eq!(alerting[0].0, "api"); - assert_eq!(alerting[1], ("worker".to_string(), 1.0)); - assert!(answers[2].is_empty(), "{:?}", answers[2]); - let [(labels, value)] = answers[3].as_slice() else { - panic!("{:?}", answers[3]); - }; - assert_eq!( - labels.keys().collect::>(), - ["instance", "job", "service"], - "{labels:?}" - ); - assert!((value - 0.98).abs() < 1e-9, "{value}"); - } -} - // ── PromQL absent() (issue #895) ───────────────────────────────────────────── - -/// The Prometheus stale marker, which Remote Write carries when a series ends. -const STALE_MARKER: f64 = f64::from_bits(0x7ff0_0000_0000_0002); - -/// The answers of the exact plan of PromQL `query` over `samples` of `m` -/// (6 series, every 10 s), with the query-time reference's. -async fn exact_answers(query: &str, samples: &[RawDataSample]) -> Vec { - let plan_json = plan_json_for( - exact_workload(&[query], ""), - &DeploymentCapabilities::default(), - ) - .await; - let run = run_data_plane(&plan_json, samples, EVALUATION_MS, &mut SummaryStore::new()); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - run.answers_per_query[0].clone() -} - -/// A sample of `m` at `ago_ms` before `EVALUATION_MS` with `labels`. -fn m_sample(labels: &[(&str, &str)], ago_ms: i64, value: f64) -> RawDataSample { - let mut sample = sample("m", "1", EVALUATION_MS - ago_ms, value); - for (name, label) in labels { - sample.labels.insert(name.to_string(), label.to_string()); - } - sample -} - -/// `absent(v)` answers as Prometheus' `funcAbsent`: one sample of value 1 -/// when `v` selects no series at the evaluation time, labelled with the -/// selector's equality matchers (`job`, not the negated `instance` and -/// `env`); no sample when some series matches, whatever its value -/// (0 included). A series whose newest sample is the stale marker has -/// ended, so `absent` answers 1 again. -#[tokio::test] -async fn absent_answers_as_prometheus_through_the_installed_plan() { - let query = r#"absent(m{job="a",instance!="3",env!="prod"})"#; - let one = vec![(BTreeMap::from([("job".to_string(), "a".to_string())]), 1.0)]; - // No sample at all. - assert_eq!(exact_answers(query, &[]).await, one); - // Only series the matchers reject: job b, env prod, instance 3. - let rejected = [ - m_sample(&[("job", "b")], 1_000, 1.0), - m_sample(&[("job", "a"), ("env", "prod")], 1_000, 1.0), - { - let mut other = m_sample(&[("job", "a")], 1_000, 1.0); - other.labels.insert("instance".into(), "3".into()); - other - }, - ]; - assert_eq!(exact_answers(query, &rejected).await, one); - // A matching series, valued 0 or not: no sample. - for value in [0.0, 5.0] { - let samples = [m_sample(&[("job", "a")], 1_000, value)]; - assert_eq!(exact_answers(query, &samples).await, vec![], "{value}"); - } - // The last matching series ended (its newest sample is stale). - let ended = [ - m_sample(&[("job", "a")], 5_000, 5.0), - m_sample(&[("job", "a")], 1_000, STALE_MARKER), - ]; - assert_eq!(exact_answers(query, &ended).await, one); -} - -/// `absent(m)` (no matcher) synthesizes a label-less sample; the missing-job -/// alert shape (several equality matchers) keeps every matcher's label. -#[tokio::test] -async fn absent_labels_come_from_the_equality_matchers_only() { - assert_eq!( - exact_answers("absent(m)", &[]).await, - vec![(BTreeMap::new(), 1.0)] - ); - let alert = r#"absent(m{kubernetes_namespace="mw-content-history-reconcile-enrich",release="production",job_name="mw_content_history_reconcile_enrich"})"#; - let labels: BTreeMap = [ - ( - "kubernetes_namespace", - "mw-content-history-reconcile-enrich", - ), - ("release", "production"), - ("job_name", "mw_content_history_reconcile_enrich"), - ] - .into_iter() - .map(|(name, value)| (name.to_string(), value.to_string())) - .collect(); - assert_eq!(exact_answers(alert, &[]).await, vec![(labels.clone(), 1.0)]); - let running = [m_sample( - &labels - .iter() - .map(|(name, value)| (name.as_str(), value.as_str())) - .collect::>(), - 1_000, - 3600.0, - )]; - assert_eq!(exact_answers(alert, &running).await, vec![]); -} diff --git a/crates/planner/tests/sql_per_group_topk.rs b/crates/planner/tests/sql_per_group_topk.rs deleted file mode 100644 index d4ea3a199..000000000 --- a/crates/planner/tests/sql_per_group_topk.rs +++ /dev/null @@ -1,405 +0,0 @@ -//! SQL's per-group top-k idiom, `ROW_NUMBER() OVER (PARTITION BY city -//! ORDER BY COUNT(*) DESC) … WHERE rn <= k`, planned with -//! `plan_installable` and run with the Planner's executor: the exact plan -//! compiles and answers each city's top hosts with their counts, the same -//! rows PromQL `topk by (city)` answers over the same data, and the plan -//! Stage 3 selects under an approximate target installs (each materialized -//! view it keeps refreshes). - -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Arc; - -use asap_executor::physical_planner::compile; -use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::values::{Batch, Value}; -use asap_planner::{ - plan_installable, test_support::query_time_reference, FrontendInput, InstallablePlan, - InstallablePlanError, PlanningModels, PlanningWorkload, SqlCatalog, UserInput, -}; -use asap_types::ir::operator::Source as DataSource; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::NonASAPOp; -use futures::{executor::block_on, StreamExt}; - -use crate::time_series; -use time_series::{time_series_tables, RawDataSample}; - -/// The evaluation time of every run. -const EVALUATION_MS: i64 = 1_800_000_000_000; - -/// The issue's per-city top 5 hosts by row count over the last 5 minutes. -const TOP_5_HOSTS_PER_CITY: &str = "SELECT city, host, c FROM (SELECT city, host, COUNT(*) AS c, \ - ROW_NUMBER() OVER (PARTITION BY city ORDER BY COUNT(*) DESC) AS rn FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE GROUP BY city, host) WHERE rn <= 5"; - -/// The issue's global top 5 hosts by row count over the last 5 minutes. -const TOP_5_HOSTS: &str = "SELECT host, COUNT(*) AS c FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE GROUP BY host ORDER BY c DESC LIMIT 5"; - -/// The same per-city top 5 in PromQL, each table row a sample of the -/// series `metrics{city, host}`. -const PROMQL_TOP_5_HOSTS_PER_CITY: &str = - "topk by (city) (5, sum by (city, host) (count_over_time(metrics[5m])))"; - -/// `metrics(ts, city, host, value)`, as the issue declares it (all but -/// `ts` nullable). -fn metrics_schema() -> Schema { - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("host", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ) -} - -/// Repeating `queries` in `language` (its workload JSON), every 10 s at -/// `accuracy`, over `metrics` ingesting 666.67 rows per second from 100 -/// series: the issue's workload. -fn workload(language: &str, queries: &[&str], accuracy: &str) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {accuracy}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": {language}, "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 666.67}}}}, - "sources": {{"metrics": {{"series_count": 100}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -const EXACT: &str = r#"{"max_error": 0.0, "failure_probability": 0.0}"#; -const APPROXIMATE: &str = r#"{"max_error": 0.01, "failure_probability": 0.001}"#; - -async fn plan_sql(queries: &[&str], accuracy: &str) -> InstallablePlan { - let catalog = SqlCatalog::new().with_table("metrics", metrics_schema()); - plan_installable(UserInput::new( - &workload(r#"{"sql": "datafusion_sql"}"#, queries, accuracy), - FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin(), - )) - .await - .expect("plan_installable plans the SQL workload") - .0 -} - -async fn plan_promql(query: &str) -> InstallablePlan { - plan_installable(UserInput::new( - &workload(r#""promql""#, &[query], EXACT), - FrontendInput::Promql, - PlanningModels::builtin(), - )) - .await - .expect("plan_installable plans the PromQL workload") - .0 -} - -/// Rows per `(city, host)` in the last 5 minutes. `nyc` has seven hosts -/// with distinct counts at the 5th place (`h5` 5, `h6` and `h7` 3), so the -/// top 5 does not depend on how ties are broken; `sf` has two hosts, fewer -/// than 5. `h7` also has 100 rows from before the window, which no answer -/// counts. -const RECENT_ROWS: [(&str, &str, i64); 9] = [ - ("nyc", "h1", 9), - ("nyc", "h2", 8), - ("nyc", "h3", 7), - ("nyc", "h4", 6), - ("nyc", "h5", 5), - ("nyc", "h6", 3), - ("nyc", "h7", 3), - ("sf", "s1", 4), - ("sf", "s2", 2), -]; - -/// The table rows: per `(city, host)`, its [`RECENT_ROWS`] count one second -/// apart ending a second before the evaluation, and `h7`'s 100 older rows -/// 10 to 11.65 minutes before it. -fn rows() -> Vec<(i64, &'static str, &'static str)> { - let mut rows = Vec::new(); - for (city, host, count) in RECENT_ROWS { - for i in 0..count { - rows.push((EVALUATION_MS - 1000 * (i + 1), city, host)); - } - } - for i in 0..100 { - rows.push((EVALUATION_MS - 600_000 - 1000 * i, "nyc", "h7")); - } - rows -} - -/// The exact answer: each city's top 5 hosts by recent row count. -fn expected_top_5_per_city() -> BTreeSet<(String, String, i64)> { - let mut per_city: BTreeMap<&str, Vec<(&str, i64)>> = BTreeMap::new(); - for (city, host, count) in RECENT_ROWS { - per_city.entry(city).or_default().push((host, count)); - } - per_city - .into_iter() - .flat_map(|(city, mut hosts)| { - hosts.sort_by_key(|&(_, count)| std::cmp::Reverse(count)); - hosts - .into_iter() - .take(5) - .map(move |(host, count)| (city.to_string(), host.to_string(), count)) - }) - .collect() -} - -/// Each answer row's `city` (when it has one), `host` and the ranked count -/// (the column named `ranked`), read by name. -fn ranked_rows(batches: &[Batch], ranked: &str) -> BTreeSet<(String, String, i64)> { - batches - .iter() - .flat_map(|batch| { - let names: Vec<&str> = batch - .schema() - .fields - .iter() - .map(|f| f.name.as_str()) - .collect(); - let column = |name: &str| names.iter().position(|n| *n == name); - let (city, host, count) = ( - column("city"), - column("host").unwrap_or_else(|| panic!("a host column in {names:?}")), - column(ranked).unwrap_or_else(|| panic!("a {ranked} column in {names:?}")), - ); - batch - .to_rows() - .into_iter() - .map(move |row| { - let text = |index: usize| match &row[index] { - Value::Utf8(text) => text.to_string(), - other => panic!("a label: {other:?}"), - }; - let count = match row[count] { - Value::Int64(n) => n, - Value::Float64(x) if x.fract() == 0.0 => x as i64, - ref other => panic!("a count: {other:?}"), - }; - (city.map(text).unwrap_or_default(), text(host), count) - }) - .collect::>() - }) - .collect() -} - -/// Query `query` of `plan` with every node at query time, at -/// [`EVALUATION_MS`], with `metrics` bound to [`rows`]: its batches. -fn run_sql(plan: &InstallablePlan, query: usize) -> Vec { - let reference = plan - .dag - .with_every_node_at_query_time() - .expect("the plan runs at query time"); - let schema = reference - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { schema, .. }) => { - Some(Arc::new(schema.clone())) - } - _ => None, - }) - .expect("the plan scans the table"); - let rows = rows() - .into_iter() - .map(|(ts, city, host)| { - vec![ - Value::Timestamp(ts), - Value::Utf8(city.into()), - Value::Utf8(host.into()), - Value::Float64(1.0), - ] - }) - .collect(); - let batch = Batch::try_new(schema.clone(), rows).unwrap(); - let store = InMemoryDataStore::default() - .with_raw_data( - DataSource::Table { - table_ref: "metrics".into(), - }, - schema, - vec![batch], - ) - .unwrap(); - let roots = [reference.roots[query] as u64]; - let bound = compile(&reference, &[], &roots) - .and_then(|compiled| compiled.instantiate(Arc::new(store))) - .expect("the plan binds to the table"); - let batches = collect(bound.execute(&roots, query_scope()).unwrap().remove(0)); - batches -} - -/// The PromQL plan's query-time reference at [`EVALUATION_MS`], each table -/// row of [`rows`] a sample of `metrics{city, host}`: its batches. -fn run_promql(plan: &InstallablePlan) -> Vec { - let samples: Vec = rows() - .into_iter() - .map(|(timestamp_ms, city, host)| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "metrics".to_string()), - ("city".to_string(), city.to_string()), - ("host".to_string(), host.to_string()), - ]), - timestamp_ms, - value: 1.0, - }) - .collect(); - let reference = query_time_reference(plan).unwrap(); - let dag = &reference.dag; - let roots = [reference.dag.roots[0] as u64]; - let compiled = compile(dag, &[], &roots).expect("the executor compiles the plan"); - // The samples bound at the plan's `Scan` of `metrics`. - let instance = compiled - .instantiate(Arc::new(time_series_tables(dag, &samples))) - .unwrap(); - let batches = collect( - instance - .execute(compiled.roots(), query_scope()) - .unwrap() - .remove(0), - ); - batches -} - -fn query_scope() -> RunContext { - RunContext::new( - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - Limits::default(), - ) - .unwrap() -} - -/// Every batch of one query's `output`. -fn collect(mut output: impl futures::Stream> + Unpin) -> Vec -where - B: std::ops::Deref, - E: std::fmt::Debug, -{ - block_on(async { - let mut batches = Vec::new(); - while let Some(batch) = output.next().await { - batches.push((*batch.expect("the query runs")).clone()); - } - batches - }) -} - -/// The exact per-city top 5 compiles and answers each city's top hosts -/// with their counts, `sf` with its two. -#[tokio::test] -async fn an_exact_sql_per_city_top_5_runs() { - let plan = plan_sql(&[TOP_5_HOSTS_PER_CITY], EXACT).await; - plan.check_compiles().unwrap(); - let answered = ranked_rows(&run_sql(&plan, 0), "topk_5"); - assert_eq!(answered, expected_top_5_per_city()); -} - -/// SQL's `ROW_NUMBER` per-city top 5 and PromQL's `topk by (city)` answer -/// the same rows over the same data. -#[tokio::test] -async fn sql_and_promql_per_city_top_5_answer_alike() { - let sql = ranked_rows( - &run_sql(&plan_sql(&[TOP_5_HOSTS_PER_CITY], EXACT).await, 0), - "topk_5", - ); - let promql = ranked_rows( - &run_promql(&plan_promql(PROMQL_TOP_5_HOSTS_PER_CITY).await), - "topk_5", - ); - assert_eq!(sql, promql); - assert_eq!(sql, expected_top_5_per_city()); -} - -/// The global top 5 answers its hosts, not only their counts. -#[tokio::test] -async fn an_exact_sql_top_5_answers_its_hosts() { - let plan = plan_sql(&[TOP_5_HOSTS], EXACT).await; - let answered: BTreeSet<(String, i64)> = ranked_rows(&run_sql(&plan, 0), "topk_5") - .into_iter() - .map(|(_, host, count)| (host, count)) - .collect(); - let expected = [("h1", 9), ("h2", 8), ("h3", 7), ("h4", 6), ("h5", 5)] - .map(|(host, count)| (host.to_string(), count)); - assert_eq!(answered, BTreeSet::from(expected)); -} - -/// The issue's two queries under ε 0.01, δ 0.001: the plan Stage 3 selects -/// installs, so the executor refreshes every materialized view it keeps and -/// compiles its query run. -#[tokio::test] -async fn the_selected_approximate_plan_installs_and_compiles() { - let plan = plan_sql(&[TOP_5_HOSTS, TOP_5_HOSTS_PER_CITY], APPROXIMATE).await; - plan.check_compiles().unwrap(); -} - -/// With `rn` selected, the per-city top 5 keeps its window function, which -/// the executor does not run, so no plan of the workload compiles. Stage 3 -/// does not compile candidates; installation compiles the selected plan -/// (each view's refresh, then the query run) and refuses it with the -/// executor's reason rather than returning a plan a runtime cannot run. -#[tokio::test] -async fn a_plan_the_executor_cannot_run_is_refused_at_installation() { - let with_rn = - TOP_5_HOSTS_PER_CITY.replace("SELECT city, host, c FROM", "SELECT city, host, c, rn FROM"); - let catalog = SqlCatalog::new().with_table("metrics", metrics_schema()); - let planned = plan_installable(UserInput::new( - &workload( - r#"{"sql": "datafusion_sql"}"#, - &[TOP_5_HOSTS, &with_rn], - APPROXIMATE, - ), - FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin(), - )) - .await; - match planned { - Err(asap_planner::PlanError::Installable( - error @ (InstallablePlanError::MaterializedViewNotExecutable { .. } - | InstallablePlanError::QueryRunNotExecutable { .. }), - )) => { - let diagnostic = error.to_string(); - assert!( - diagnostic.contains("value operation has no native implementation"), - "{diagnostic}" - ); - } - other => panic!("no plan is installed: {:?}", other.map(|_| ())), - } -} - -/// A per-city top 5 the heavy-hitter `TopK` does not cover runs exactly as -/// the per-partition `Limit{Sort}`, with the query's own columns: ordered by -/// the count then the host (a tie-breaking second key), and with a SELECT -/// list that drops the partition column. -#[tokio::test] -async fn generic_per_city_top_5_forms_run_exactly() { - let tie_broken = TOP_5_HOSTS_PER_CITY.replace("COUNT(*) DESC)", "COUNT(*) DESC, host)"); - let plan = plan_sql(&[&tie_broken], EXACT).await; - assert_eq!( - ranked_rows(&run_sql(&plan, 0), "c"), - expected_top_5_per_city() - ); - let hosts_only = - TOP_5_HOSTS_PER_CITY.replace("SELECT city, host, c FROM", "SELECT host, c FROM"); - let plan = plan_sql(&[&hosts_only], EXACT).await; - let expected: BTreeSet<_> = expected_top_5_per_city() - .into_iter() - .map(|(_, host, count)| (String::new(), host, count)) - .collect(); - assert_eq!(ranked_rows(&run_sql(&plan, 0), "c"), expected); -} diff --git a/crates/planner/tests/sql_table_scans.rs b/crates/planner/tests/sql_table_scans.rs index 69dd65cf4..a148bcfc4 100644 --- a/crates/planner/tests/sql_table_scans.rs +++ b/crates/planner/tests/sql_table_scans.rs @@ -12,19 +12,17 @@ use std::sync::Arc; use asap_executor::physical_planner::compile; use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::sources::{ - DataStore, InMemoryDataStore, MaterializedViewRequest, RawDataRequest, -}; +use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; use asap_planner::{ plan_installable, DeploymentCapabilities, FrontendInput, InstallablePlan, MaterializedViewRefresh, PlanningModels, PlanningWorkload, SqlCatalog, UserInput, }; -use asap_types::ir::materialized_view::MaterializedView; + use asap_types::ir::operator::Source; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::NonASAPOp; +use asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; use futures::{executor::block_on, StreamExt}; const MINUTE_MS: i64 = 60_000; @@ -78,20 +76,6 @@ const APPROXIMATE: &str = r#"{"accuracy_metric": "relative", "max_error": 0.05, /// No approximation. const EXACT: &str = r#"{"max_error": 0.0, "failure_probability": 0.0}"#; -async fn plan(query: &str, accuracy: &str) -> InstallablePlan { - plan_all(&[query], accuracy).await -} - -async fn plan_all(queries: &[&str], accuracy: &str) -> InstallablePlan { - plan_for( - queries, - accuracy, - MINUTE_MS as u64, - &DeploymentCapabilities::default(), - ) - .await -} - /// [`plan_all`] for a deployment that keeps no raw data: every window is /// read by state maintained at ingestion time. async fn plan_without_raw_data(queries: &[&str], accuracy: &str) -> InstallablePlan { @@ -147,74 +131,6 @@ fn rows() -> Vec> { .collect() } -/// Runs query 0 of `plan`, every node at query time (its reference plan, -/// which reads the table's rows wherever the plan keeps state), at -/// [`EVALUATION_MS`] with `http` bound to [`rows`]: its output field types -/// and its rows. A `WHERE ts >= now() - INTERVAL` window is a `TimeRange` -/// the executor runs as a filter on `ts` against the evaluation time. -fn run(plan: &InstallablePlan) -> (Vec, Vec>) { - run_query(plan, 0) -} - -/// [`run`] for query `query` of `plan`. -fn run_query(plan: &InstallablePlan, query: usize) -> (Vec, Vec>) { - let reference = plan - .dag - .with_every_node_at_query_time() - .expect("the plan runs at query time"); - // The table's rows in the schema the plan's scan reads (its columns - // qualified by the table). - let schema = reference - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { schema, .. }) => { - Some(Arc::new(schema.clone())) - } - _ => None, - }) - .expect("the plan scans the table"); - let batch = Batch::try_new(schema.clone(), rows()).unwrap(); - let store = InMemoryDataStore::default() - .with_raw_data( - Source::Table { - table_ref: "http".into(), - }, - schema, - vec![batch], - ) - .unwrap(); - let roots = [reference.roots[query] as u64]; - let bound = compile(&reference, &[], &roots) - .and_then(|compiled| compiled.instantiate(Arc::new(store))) - .expect("the plan binds to the table"); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - Limits::default(), - ) - .unwrap(); - block_on(async { - let mut output = bound.execute(&roots, context).unwrap().remove(0); - let mut fields = None; - let mut rows = Vec::new(); - while let Some(batch) = output.next().await { - let batch = batch.expect("the query runs"); - fields.get_or_insert_with(|| { - batch - .schema() - .fields - .iter() - .map(|f| f.dtype.clone()) - .collect() - }); - rows.extend(batch.to_rows()); - } - (fields.unwrap_or_default(), rows) - }) -} - /// The answer rows by their first column, each value as it prints (its /// type and value). fn by_key(rows: Vec>) -> BTreeMap { @@ -235,17 +151,6 @@ fn shown(value: &Value) -> String { } } -/// The planned output type of query 0's root. -fn planned_output(plan: &InstallablePlan) -> Vec { - let root = plan.dag.roots[0]; - let node = plan.dag.nodes.iter().find(|n| n.id == root).unwrap(); - node.output_schema - .fields - .iter() - .map(|f| f.dtype.clone()) - .collect() -} - const DISTINCT_USERS_LAST_5_MINUTES: &str = "SELECT service, COUNT(DISTINCT user_id) AS value \ FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service"; @@ -258,164 +163,11 @@ fn expected_distinct_users() -> BTreeMap { .collect() } -#[tokio::test] -async fn a_distinct_count_over_an_int64_id_in_a_now_window_runs_from_a_sketch() { - let plan = plan(DISTINCT_USERS_LAST_5_MINUTES, APPROXIMATE).await; - let selected = format!("{:?}", plan.dag.nodes); - assert!( - selected.contains("SummaryAgg"), - "a distinct-count sketch answers it: {selected}" - ); - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - // The sketch is exact below its capacity. - assert_eq!(by_key(rows), expected_distinct_users()); -} - -#[tokio::test] -async fn an_exact_distinct_count_over_an_int64_id_in_a_now_window_runs() { - let plan = plan(DISTINCT_USERS_LAST_5_MINUTES, EXACT).await; - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - assert_eq!(by_key(rows), expected_distinct_users()); -} - -#[tokio::test] -async fn now_is_the_evaluation_time() { - let plan = plan( - "SELECT service, COUNT(*) AS value FROM http \ - WHERE ts > now() - INTERVAL '1 minute' GROUP BY service", - EXACT, - ) - .await; - let (_, rows) = run(&plan); - // The 60 rows after the minute before the evaluation: 20 per service. - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(20)))) - .collect(); - assert_eq!(by_key(rows), expected); -} - -/// SQL `percentile_cont` of `values`: linear interpolation at rank -/// `q * (n - 1)` over the sorted values. -fn percentile_cont(q: f64, mut values: Vec) -> f64 { - values.sort_by(f64::total_cmp); - let rank = q * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - values[low] + (values[high] - values[low]) * (rank - low as f64) -} - -#[tokio::test] -async fn an_exact_grouped_sql_percentile_runs() { - let plan = plan( - "SELECT service, approx_percentile_cont(latency, 0.99) AS value FROM http \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service", - EXACT, - ) - .await; - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - // Each service's latencies `i` over the last 5 minutes. The window is a - // range selector's `(T - 5m, T]`, so the row stamped exactly `T - 5m` - // (i = 899) is not read: i >= 900. - let expected: BTreeMap = (0..3i64) - .map(|s| { - let latencies = (900..1200) - .filter(|i| i % 3 == s) - .map(|i| i as f64) - .collect(); - ( - format!("s{s}"), - shown(&Value::Float64(percentile_cont(0.99, latencies))), - ) - }) - .collect(); - assert_eq!(by_key(rows), expected); -} - -/// A global percentile over an empty window is NULL (SQL), and the plan -/// types it nullable. -#[tokio::test] -async fn an_exact_global_sql_percentile_of_no_row_is_null() { - let plan = plan( - "SELECT approx_percentile_cont(latency, 0.5) AS value FROM http \ - WHERE ts > now()", - EXACT, - ) - .await; - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - assert_eq!(rows.len(), 1); - assert!(matches!(rows[0][0], Value::Null), "{:?}", rows[0][0]); -} - -/// A 5 and a 15 minute distinct count over one table: the plan builds them -/// in tumbling windows, each tumbling window a `TimeRange` over a shifted table `Scan`, -/// which the executor runs as filters on `ts`. Both answer exactly (the -/// sketch is exact below its capacity). -#[tokio::test] -async fn distinct_counts_over_5_and_15_minute_windows_run_from_tumbling_windows() { - let fifteen = DISTINCT_USERS_LAST_5_MINUTES.replace("5 minutes", "15 minutes"); - let plan = plan_all(&[DISTINCT_USERS_LAST_5_MINUTES, &fifteen], APPROXIMATE).await; - let selected = format!("{:?}", plan.dag.nodes); - assert!( - selected.contains("TimeShift"), - "tumbling windows: {selected}" - ); - assert_eq!(by_key(run_query(&plan, 0).1), expected_distinct_users()); - // The last 15 minutes (i >= 300): every user `i % 50` of each service - // (3 and 50 are coprime) and its 7 recent ones. - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(57)))) - .collect(); - assert_eq!(by_key(run_query(&plan, 1).1), expected); -} - -/// Exact row counts over 5 and 15 minutes: a count tells a row read twice -/// or missed at a tumbling window's edge, which a distinct count cannot. Each tumbling window is -/// `(T - s - w, T - s]`, so the 15-minute tumbling windows read every row after -/// `T - 15m` exactly once: 300 per service (the row at `T - 15m` is not -/// read), and 100 in the last 5 minutes. -#[tokio::test] -async fn row_counts_over_5_and_15_minute_windows_read_each_row_once() { - let five = "SELECT service, COUNT(*) AS value FROM http \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service"; - let fifteen = five.replace("5 minutes", "15 minutes"); - let plan = plan_all(&[five, &fifteen], EXACT).await; - let selected = format!("{:?}", plan.dag.nodes); - assert!( - selected.contains("TimeShift"), - "tumbling windows: {selected}" - ); - for (query, rows_per_service) in [(0, 100), (1, 300)] { - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(rows_per_service)))) - .collect(); - assert_eq!(by_key(run_query(&plan, query).1), expected, "query {query}"); - } -} - /// Runs `ids` of `dag` in `scope` with each of `inputs` bound to the node /// its key names and `table` bound as `http`: the batches of each id's /// output. fn run_bound( - dag: &asap_types::ir::physical_export::PhysicalASAPDAG, + dag: &asap_types::plan::physical_asap_dag::PhysicalASAPDAG, inputs: BTreeMap, table: &Batch, ids: &[u64], @@ -546,7 +298,7 @@ fn run_data_plane(plan: &InstallablePlan) -> Vec>> { .collect() } -/// A 5 and a 15 minute distinct count kept in tumbling windows at ingestion +/// A 5 and a 15 minute distinct count materialized in tumbling windows at ingestion /// time (the deployment keeps no raw data): each tumbling window is built in an ingestion run from the table rows of its /// window, and both queries answer exactly from the tumbling windows (the sketch is /// exact below its capacity), as the query-time reference does. @@ -641,106 +393,3 @@ async fn a_window_equal_to_its_interval_is_built_at_ingestion_time_from_table_ro .collect(); assert_eq!(by_key(run_data_plane(&plan).remove(0)), expected); } - -/// Each run's requested `(start, end]` time range, in order. -type RequestedTimeRanges = std::sync::Mutex>>; - -/// A store whose `http` table records each run's requested time range and -/// returns every row (it ignores the request). -struct RecordingStore { - rows: InMemoryDataStore, - requested_time_ranges_ms: Arc, -} -impl DataStore for RecordingStore { - fn read_raw_data( - &self, - source: &Source, - request: &RawDataRequest, - context: RunContext, - ) -> Result, asap_executor::Error> { - self.requested_time_ranges_ms - .lock() - .unwrap() - .push(request.time_range_ms); - self.rows.read_raw_data(source, request, context) - } - fn read_materialized_view( - &self, - view: &MaterializedView, - request: &MaterializedViewRequest, - context: RunContext, - ) -> Result, asap_executor::Error> { - self.rows.read_materialized_view(view, request, context) - } -} - -// The store's `http` table of a `ts >= now() - INTERVAL '5 minutes'` query is -// asked for `(T - 5m, T]` at evaluation time `T`; it returns all 20 minutes -// of rows, and the answer is the one over the window. -#[tokio::test] -async fn a_data_store_is_asked_for_the_query_window() { - // The user's query, planned exactly over the `http` table. - let plan = plan( - "SELECT service, COUNT(*) AS value FROM http \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service", - EXACT, - ) - .await; - // The plan with every node at query time: it reads the table at the Scan. - let reference = plan.dag.with_every_node_at_query_time().unwrap(); - // The schema the plan's Scan reads. - let schema = reference - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { schema, .. }) => { - Some(Arc::new(schema.clone())) - } - _ => None, - }) - .unwrap(); - // Each run's requested time range, as the store saw it. - let requested_time_ranges_ms = Arc::new(RequestedTimeRanges::default()); - // The deployment's store: `http` → its rows, recording each request. - let store = RecordingStore { - rows: InMemoryDataStore::default() - .with_raw_data( - Source::Table { - table_ref: "http".into(), - }, - schema.clone(), - vec![Batch::try_new(schema, rows()).unwrap()], - ) - .unwrap(), - requested_time_ranges_ms: requested_time_ranges_ms.clone(), - }; - // Compiled once (no reader opened), then instantiated for this run. - let roots = [reference.roots[0] as u64]; - let compiled = compile(&reference, &[], &roots).unwrap(); - let bound = compiled.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - Limits::default(), - ) - .unwrap(); - // The answer rows: each service and its count. - let rows = block_on(async { - let mut output = bound.execute(&roots, context).unwrap().remove(0); - let mut rows = Vec::new(); - while let Some(batch) = output.next().await { - rows.extend(batch.unwrap().to_rows()); - } - rows - }); - assert_eq!( - *requested_time_ranges_ms.lock().unwrap(), - vec![Some((EVALUATION_MS - 5 * MINUTE_MS, EVALUATION_MS))] - ); - // The 300 rows of the window, 100 per service. - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(100)))) - .collect(); - assert_eq!(by_key(rows), expected); -} diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index bc7658d28..e99beba48 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -1,23 +1,22 @@ //! Structurally identical summary producers chosen by different queries are -//! shared: one `Rc` across their plans, planned by Pass 2 +//! shared: one node of the plan across their roots, planned by Pass 2 //! (ASAP-aware CSE). Some older cost-selection tests stay ignored: with the built-in cost model, //! plan selection still selects an exact plan for at least one of their //! queries (#580). -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; +use asap_types::ir::{ASAPOp, OperatorNode}; use std::rc::Rc; -use asap_accuracy::test_support::{default_size_params, DEFAULT_DELTA}; -use asap_frontend_promql::{lower_promql_query_workload, promql_metric_names}; +use asap_frontend_promql::promql_metric_names; use asap_frontend_sql::SqlCatalog; -use asap_plan_selection::reference_pricer::WholePlanSelection; -use asap_plan_selection::RejectionCause; + use asap_planner::{e2e_plan, FrontendInput, UserInput}; -use asap_planner::{PlanOutput, PlanningModels, QueryPlan}; -use asap_types::ir::operator::agg_intent::default_quantile; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; +use asap_planner::{PlanOutput, PlanningModels}; +use asap_types::ir::QueryRoot; + +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; + use asap_types::workload::SqlDialect; use asap_types::workload::{ DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, @@ -68,6 +67,7 @@ fn promql_workload(queries: &[(&str, f64)]) -> PlanningWorkload { data_workload: Some(DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1.0, + max_out_of_order_ms: None, }, ..DataWorkload::sampled_every( queries @@ -86,166 +86,6 @@ async fn plan_promql(queries: &[(&str, f64)]) -> PlanOutput { e2e_plan(input).await.expect("workload plans") } -/// Explicit exact, all-KLL, and all-DDS assignments for composition checks. -/// These fixtures test candidate support and rejection independently of which -/// candidates a bounded search happens to visit. -fn composed_summary_candidates( - queries: &[(&str, f64)], -) -> ( - Vec, - WholePlanSelection, -) { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = promql_workload(queries); - let roots = lower_promql_query_workload(&workload).expect("lowers"); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut assignments = vec![vec![0; inventory.targets.len()]]; - for family in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let choices: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - *candidate == HowToComputeAggregateMeasure::Summary(family) - }) - .unwrap_or(0) - }) - .collect(); - assert_eq!( - choices.iter().filter(|choice| **choice != 0).count(), - 2, - "both quantile references must explicitly use {family:?}" - ); - assignments.push(choices); - } - let entries = workload.query_workload.entries().collect::>(); - let data = workload.data_workload.as_ref().unwrap(); - let models = PlanningModels::builtin(); - let candidates: Vec<_> = assignments - .into_iter() - .enumerate() - .map(|(index, choices)| { - let roots: Vec<_> = compose_logical_candidate(&inventory, &choices).unwrap(); - let mut candidate = - asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - "explicit-composition", - &roots, - &entries, - data, - models.deployment_capabilities.query_time_results_retained, - &mut |_, _, _| Some(asap_types::physical::Materialization::NotMaterialized), - ) - .unwrap() - .expect("supported physical computation"); - candidate.id = format!("composition-{index}"); - candidate - }) - .collect(); - let selection = select_plan(&candidates, &entries, data, models) - .expect("the exact candidate meets the requested accuracy"); - (candidates, selection) -} - -/// Construct a legal shared-KLL candidate explicitly, independent of the -/// plan the search selects. -fn shared_kll_candidate(workload: &PlanningWorkload) -> asap_types::physical::PhysicalWorkloadPlan { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::physical::Materialization; - let roots = lower_promql_query_workload(workload).unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut choices = vec![0; inventory.targets.len()]; - for index in 0..choices.len() { - choices[index] = (1..inventory.targets[index].choice_count()).find(|choice| { - let mut trial = vec![0; choices.len()]; - trial[index] = *choice; - compose_logical_candidate(&inventory, &trial).unwrap().iter().any(|root| { - root.operator_refs().into_iter().flat_map(OperatorNode::reachable).any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) if matches!(kind.params(), SketchParams::Kll { .. })) - }) - }) - }).expect("KLL implementation exists"); - } - let roots: Vec<_> = compose_logical_candidate(&inventory, &choices).unwrap(); - // Every identical sub-DAG of the two queries made one node. - let roots: Vec<_> = asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .into_iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, node), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect(); - let mut physical = - asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - "shared-kll", - &roots, - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - false, - &mut |_, _, _| Some(Materialization::NotMaterialized), - ) - .unwrap() - .unwrap(); - physical.id = "shared-kll".into(); - select_plan( - std::slice::from_ref(&physical), - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin(), - ) - .expect("shared candidate meets both accuracy targets"); - physical -} - -fn shared_kll_output(workload: &PlanningWorkload) -> PlanOutput { - let candidate = shared_kll_candidate(workload); - PlanOutput::new( - candidate - .roots - .into_iter() - .enumerate() - .map(|(entry_index, root)| { - let QueryRoot::Operator(root) = root else { - panic!("operator fixture") - }; - QueryPlan { entry_index, root } - }) - .collect(), - ) -} - -/// Whether `root` reads a sketch state. -fn reads_sketch(root: &Rc) -> bool { - OperatorNode::reachable(root).into_iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) - }) -} - async fn plan_sql(queries: &[&str]) -> PlanOutput { let workload = PlanningWorkload { query_workload: QueryWorkload { @@ -281,9 +121,24 @@ async fn plan_sql(queries: &[&str]) -> PlanOutput { e2e_plan(input).await.expect("workload plans") } -/// Every summary state (`SummaryAgg`) each plan reaches, in traversal order. -fn plan_states(plan: &QueryPlan) -> Vec> { - OperatorNode::reachable(&plan.root) +/// Each operator query's root of `output`'s plan, as IR, in entry order. +fn operator_roots(output: &PlanOutput) -> Vec> { + output + .plan + .query_roots() + .unwrap() + .into_iter() + .filter_map(|root| match root { + QueryRoot::Operator(node) => Some(node), + QueryRoot::Scalar(_) => None, + }) + .collect() +} + +/// Every summary state (`SummaryAgg`) a query's root reaches, in traversal +/// order. +fn plan_states(root: &Rc) -> Vec> { + OperatorNode::reachable(root) .into_iter() .filter(|node| { matches!( @@ -296,12 +151,11 @@ fn plan_states(plan: &QueryPlan) -> Vec> { /// Every summary state each plan reaches; each plan selects at least one. fn states(output: &PlanOutput) -> Vec>> { - output - .plans + operator_roots(output) .iter() - .map(|plan| { - let states = plan_states(plan); - assert!(!states.is_empty(), "{:?}", plan.root); + .map(|root| { + let states = plan_states(root); + assert!(!states.is_empty(), "{root:?}"); states }) .collect() @@ -319,8 +173,8 @@ fn same_states(states: &[Vec>]) -> bool { /// The summary states a consumer would run, deduplicated by pointer. fn unique_deployments(output: &PlanOutput) -> usize { let mut seen: Vec<*const OperatorNode> = Vec::new(); - for plan in &output.plans { - for state in plan_states(plan) { + for root in operator_roots(output) { + for state in plan_states(&root) { let ptr = Rc::as_ptr(&state); if !seen.contains(&ptr) { seen.push(ptr); @@ -330,153 +184,6 @@ fn unique_deployments(output: &PlanOutput) -> usize { seen.len() } -/// An explicitly connected KLL candidate shares one physical producer and -/// satisfies both outputs' accuracy contracts. -#[test] -fn quantiles_with_equal_params_share_one_producer() { - let workload = promql_workload(&[ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[5m])", 0.01), - ]); - let cheapest = shared_kll_candidate(&workload); - let states: Vec>> = cheapest - .roots - .iter() - .map(|root| { - root.operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect() - }) - .collect(); - assert_eq!(states[0].len(), 1, "{}", cheapest.id); - assert!(same_states(&states), "{}", cheapest.id); -} - -/// Shared producer identity survives physicalization for a high-volume source; -/// this checks feasibility, independently of the cost optimizer's winner. -#[tokio::test] -async fn quantiles_with_equal_params_deploy_one_producer() { - let mut workload = promql_workload(&[ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[5m])", 0.01), - ]); - let data = workload.data_workload.as_mut().unwrap(); - data.arrival = DataArrival::Ingesting { - rows_per_second: 666_666.67, - }; - let lat = data.sources.get_mut("lat").unwrap(); - lat.series_count = Some(1_000_000); - lat.sample_interval = Some(DurationMs(1_500)); - let output = shared_kll_output(&workload); - assert!(same_states(&states(&output))); - assert!(!Rc::ptr_eq(&output.plans[0].root, &output.plans[1].root)); - assert_eq!(unique_deployments(&output), 1); -} - -/// A different window or label selector is a different producer, even when -/// one query asks for a stricter accuracy than the other. -#[tokio::test] -#[ignore = "plan selection still selects an exact plan for at least one query: with the built-in cost model its query-time summary costs more than exact (#580)"] -async fn different_producers_are_not_shared() { - for queries in [ - [ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[10m])", 0.01), - ], - [ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[10m])", 0.001), - ], - [ - ("quantile_over_time(0.5, lat{job=\"a\"}[5m])", 0.01), - ("quantile_over_time(0.99, lat{job=\"b\"}[5m])", 0.001), - ], - [ - ("quantile_over_time(0.5, lat{job=\"a\"}[5m])", 0.01), - ("quantile_over_time(0.99, lat{job=\"b\"}[5m])", 0.01), - ], - ] { - let output = plan_promql(&queries).await; - assert!(!same_states(&states(&output)), "{queries:?}"); - assert_eq!(unique_deployments(&output), 2, "{queries:?}"); - for (plan, (_, epsilon)) in output.plans.iter().zip(queries) { - assert_eq!(kll_k(plan), kll_k_for(epsilon), "{queries:?}"); - } - } -} - -/// The KLL `k` of the one state a plan deploys. -fn kll_k(plan: &QueryPlan) -> u32 { - let states = plan_states(plan); - let [deployment] = states.as_slice() else { - panic!("one state: {:?}", states.len()); - }; - let asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = &deployment.operator - else { - panic!("sketch state: {:?}", deployment.operator); - }; - let SketchParams::Kll { k, .. } = kind.params() else { - panic!("KLL state: {kind:?}"); - }; - *k -} - -/// The KLL `k` sized for `epsilon`. -fn kll_k_for(epsilon: f64) -> u32 { - let SketchParams::Kll { k, .. } = default_size_params( - SketchAlgorithm::Kll, - &default_quantile(0.5), - epsilon, - DEFAULT_DELTA, - ) else { - unreachable!() - }; - k -} - -/// p50 at ε=0.01 and p99 at ε=0.001 over the same input share one KLL sized -/// for the strictest consumer; each reader's guarantee meets its own target. -#[tokio::test] -#[ignore = "plan selection still selects an exact plan for at least one query: with the built-in cost model its query-time summary costs more than exact (#580)"] -async fn quantiles_share_one_producer_sized_for_the_strictest_consumer() { - let p50 = ("quantile_over_time(0.5, lat[5m])", 0.01); - let p99 = ("quantile_over_time(0.99, lat[5m])", 0.001); - assert!(kll_k_for(0.001) > kll_k_for(0.01)); - - let output = plan_promql(&[p50, p99]).await; - assert!(same_states(&states(&output))); - assert_eq!(unique_deployments(&output), 1); - for (plan, (_, epsilon)) in output.plans.iter().zip([p50, p99]) { - assert_eq!(kll_k(plan), kll_k_for(0.001)); - let guarantee = plan.root.guarantee.as_ref().expect("certified"); - assert!( - guarantee.spec.error.evaluate().unwrap() <= epsilon, - "{guarantee:?}" - ); - } - - // Alone, the looser query keeps its own, smaller KLL. - let alone = plan_promql(&[p50]).await; - assert_eq!(kll_k(&alone.plans[0]), kll_k_for(0.01)); -} - -/// Cross-series quantiles name their KLL state after the input column, not the -/// quantile, so p50 and p99 over one selector share it. -#[tokio::test] -async fn cross_series_p50_and_p99_share_one_producer() { - let output = shared_kll_output(&promql_workload(&[ - ("quantile(0.5, lat)", 0.01), - ("quantile(0.99, lat)", 0.01), - ])); - assert!(same_states(&states(&output))); - assert_eq!(unique_deployments(&output), 1); -} - /// Two identical ungrouped queries: their identical producers (rate, then /// sum) are shared. #[tokio::test] @@ -511,12 +218,10 @@ async fn sql_p50_and_p99_share_one_producer() { let output = plan_sql(&[p50, p99]).await; assert!(same_states(&states(&output))); assert_eq!(unique_deployments(&output), 1); - let names: Vec<_> = output - .plans + let names: Vec<_> = operator_roots(&output) .iter() - .map(|plan| { - plan.root - .schema + .map(|root| { + root.schema .fields .iter() .map(|field| field.name.clone()) @@ -547,127 +252,9 @@ async fn sql_p50_and_p99_share_one_producer() { } } -/// The operator sub-DAG a standalone scalar query reads is planned like an -/// operator root (#511 §2.3): the scalar root is rebuilt over its selected -/// sub-DAG, and entries keep their order around a literal scalar query. -#[tokio::test] -async fn scalar_queries_read_planned_sub_dags() { - let output = plan_promql(&[ - ("1", 0.01), - ("scalar(quantile(0.99, rate(x[5m])))", 0.01), - ("quantile(0.99, rate(x[5m]))", 0.01), - ]) - .await; - assert_eq!(output.entry_indices(), vec![0, 1, 2]); - let scalar_entries: Vec<_> = output.scalar_roots.iter().map(|(i, _)| *i).collect(); - assert_eq!(scalar_entries, vec![0, 1]); - assert_eq!(output.plans.len(), 1); - assert_eq!(output.plans[0].entry_index, 2); - - assert_eq!(output.scalar_plans.len(), 1); - let read = &output.scalar_plans[0]; - assert_eq!(read.entry_index, 1); - let refs = output.scalar_roots[1].1.operator_refs(); - assert_eq!(refs.len(), 1); - assert!(Rc::ptr_eq(refs[0], &read.root)); - - // The export keys each scalar query by its entry and points the scalar - // read at an exported node. - let dag = output.physical_asap_dag().unwrap(); - let keys: Vec<_> = dag.scalar_roots.iter().map(|(i, _)| *i).collect(); - assert_eq!(keys, vec![0, 1]); - assert_eq!(dag.roots.len(), 1); - let [read] = dag.scalar_roots[1].1.operator_refs()[..] else { - panic!("one read"); - }; - assert!(dag.nodes.iter().any(|node| node.id == *read)); -} - -/// A combination of approximate values keeps its summary candidates: each -/// reference of `scalar(q99) - scalar(q50)` is planned against the entry's -/// target, so a candidate reads a sketch in both. A difference of rank (KLL) -/// or relative (DDSketch) errors composes to no guarantee in the MVP (#669) -/// and the built-in models have no empirical output-level evidence, so plan -/// selection rejects it with that reason. -#[test] -fn combined_scalar_references_keep_summary_candidates() { - let (candidates, selection) = composed_summary_candidates(&[( - "scalar(quantile(0.99, rate(x[5m]))) - scalar(quantile(0.5, rate(x[5m])))", - 0.01, - )]); - for both in candidates.iter().skip(1) { - let refs = both.roots[0].operator_refs(); - assert_eq!(refs.len(), 2); - assert!( - refs.into_iter().all(reads_sketch), - "both references use the explicit sketch family" - ); - let rejection = selection - .rejected - .iter() - .find(|r| r.id == both.id) - .expect("not selected"); - assert_eq!( - rejection.cause, - RejectionCause::NoAccuracyGuarantee, - "{rejection:?}" - ); - assert!( - rejection.reason.contains("no guarantee"), - "{}", - rejection.reason - ); - } - assert_eq!( - selection.selected, candidates[0].id, - "the exact assignment is selected" - ); -} - -/// Candidate generation supports summary implementations of the vector -/// operands, but a difference or a comparison of -/// estimates composes to no guarantee in the MVP (#669; a comparison's band -/// guarantee is later work) and the built-in models have no empirical -/// output-level evidence (plan selection, #576), so it is not selected -/// automatically: the exact plan is. -#[tokio::test] -async fn combined_vector_quantiles_keep_unselected_summary_candidates() { - for query in [ - "quantile(0.99, rate(x[5m])) - quantile(0.5, rate(x[5m]))", - "quantile(0.99, rate(x[5m])) > quantile(0.5, rate(x[5m]))", - ] { - let (candidates, selection) = composed_summary_candidates(&[(query, 0.01)]); - for summary in candidates.iter().skip(1) { - let QueryRoot::Operator(root) = &summary.roots[0] else { - panic!("{query}: operator root"); - }; - assert!(reads_sketch(root), "{query}: explicit summary candidate"); - assert!(root.guarantee.is_none(), "{query}"); - let rejection = selection - .rejected - .iter() - .find(|rejection| rejection.id == summary.id) - .expect("uncertified vector computation is rejected"); - assert_eq!( - rejection.cause, - RejectionCause::NoAccuracyGuarantee, - "{query}: {rejection:?}" - ); - assert!(rejection.reason.contains("no guarantee"), "{rejection:?}"); - } - assert_eq!( - selection.selected, candidates[0].id, - "{query}: exact assignment selected" - ); - - let output = plan_promql(&[(query, 0.01)]).await; - assert!(!reads_sketch(&output.plans[0].root), "{query}"); - } -} - /// Front ends build structurally identical references as separate nodes. /// The selected graph may reuse a producer or choose separate physical -/// candidates. Every distinct selected reference must be exported exactly once. +/// candidates; the scalar root reads at least the distinct ones. #[tokio::test] async fn identical_scalar_references_satisfy_the_plan_contract() { for (query, distinct) in [ @@ -678,91 +265,12 @@ async fn identical_scalar_references_satisfy_the_plan_contract() { ), ] { let output = plan_promql(&[(query, 0.01)]).await; - let references: std::collections::HashSet<_> = output.scalar_roots[0] + let references: std::collections::HashSet<_> = output.plan.scalar_roots[0] .1 .operator_refs() .into_iter() - .map(Rc::as_ptr) + .copied() .collect(); - assert_eq!(output.scalar_plans.len(), references.len(), "{query}"); assert!(references.len() >= distinct, "{query}"); - let (_, expr) = &output.scalar_roots[0]; - for node in expr.operator_refs() { - assert!( - output - .scalar_plans - .iter() - .any(|read| Rc::ptr_eq(&read.root, node)), - "{query}" - ); - } } } - -/// A root whose selected candidate is an exact state is a query result: it -/// is read through `FinalizeExactAccumulator`, so its output is plain values, -/// not accumulator state. -#[tokio::test] -async fn exact_state_roots_are_finalized() { - for query in [ - "sum_over_time(m[1m])", - "sum(rate(x[5m]))", - "sum(sum_over_time(m[1m]) + sum_over_time(n[1m]))", - ] { - let output = plan_promql(&[(query, 0.01)]).await; - let root = &output.plans[0].root; - assert!( - matches!(root.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })), - "{query}: {root:#?}" - ); - let dag = output.physical_asap_dag().unwrap(); - let root = dag - .nodes - .iter() - .find(|node| node.id == dag.roots[0]) - .unwrap(); - assert!( - root.output_schema - .fields - .iter() - .all(|field| field.is_plain()), - "{query}: {:?}", - root.output_schema - ); - // With nothing materialized, the root and the exact state it reads - // run at query time. - use asap_types::ir::properties::{DataPrimitive, ExecutionDataState, ExecutionTiming}; - assert_eq!(root.output_state, ExecutionDataState::QUERY_ROWS, "{query}"); - let state = dag - .edges - .iter() - .find(|edge| edge.consumer == root.id) - .map(|edge| edge.producer) - .unwrap(); - let state = dag.nodes.iter().find(|node| node.id == state).unwrap(); - assert_eq!( - state.output_state, - ExecutionDataState { - timing: ExecutionTiming::QueryTime, - primitive: DataPrimitive::SummaryState, - }, - "{query}" - ); - } -} - -/// The reference pricer over `candidates`, every candidate kept. -fn select_plan( - candidates: &[asap_types::physical::PhysicalWorkloadPlan], - entries: &[asap_types::workload::QueryWorkloadEntry], - data: &asap_types::workload::DataWorkload, - models: PlanningModels<'_>, -) -> Result { - asap_plan_selection::reference_pricer::select_among_plans( - candidates, - entries, - data, - &models.accuracy_check_inputs(), - models.cost_inputs, - ) -} diff --git a/crates/planner/tests/time_series/mod.rs b/crates/planner/tests/time_series/mod.rs index b6e6881ae..43a799a2d 100644 --- a/crates/planner/tests/time_series/mod.rs +++ b/crates/planner/tests/time_series/mod.rs @@ -10,9 +10,9 @@ use std::sync::Arc; use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; use asap_types::ir::operator::Source; -use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; use asap_types::ir::schema::{DataType, FieldDataType, Schema, PROMQL_SERIES_IDENTITY}; use asap_types::ir::NonASAPOp; +use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; /// One raw data sample as it arrives, e.g. one Prometheus Remote Write sample. #[derive(Debug, Clone, PartialEq)] diff --git a/crates/types/src/ir/properties/guarantee.rs b/crates/types/src/accuracy/guarantee.rs similarity index 98% rename from crates/types/src/ir/properties/guarantee.rs rename to crates/types/src/accuracy/guarantee.rs index ead2871e1..e432b41df 100644 --- a/crates/types/src/ir/properties/guarantee.rs +++ b/crates/types/src/accuracy/guarantee.rs @@ -8,8 +8,8 @@ //! *vocabulary* that fixes that — the typed metric, the symbolic bound and //! failure-probability expressions, the provenance trail, and the typed //! rejection reasons. The *algebra* that composes these (the built-in -//! conservative rules and budget allocation) and the deployment's -//! pluggable `AccuracyModel` trait live one layer up in `asap_accuracy`: +//! conservative rules and budget allocation) and the deployment's measured +//! accuracy (`AccuracyEvidence`) live one layer up in `asap_accuracy`: //! this crate defines the shapes, the planning crates decide. //! //! ## What a guarantee says @@ -52,8 +52,7 @@ use serde::{Deserialize, Serialize}; -use crate::ir::properties::AccuracyTarget; -pub use crate::ir::properties::{AccuracyMetric, AccuracyScope, AccuracySpec, NormScale}; +use super::target::{AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget}; /// A symbolic, serializable error-bound expression. Non-negative real /// arithmetic only — there is no subtraction, so a bound can never be @@ -468,9 +467,8 @@ impl AccuracyGuarantee { /// never fails (`δ = 0`) holds for every value at once; an unbound `n` /// (the whole result of an unbound `G`) leaves the failure probability /// symbolic in [`VALUES_PER_RESULT`]. A guarantee over any other scope - /// is returned as it is: the check then accepts it only for a scope it - /// covers ([`AccuracyScope::covers`]). Plan selection uses it to check a - /// target over `scope`. + /// is returned as it is. Plan selection uses it to check a target over + /// `scope`. pub fn over_scope(&self, scope: AccuracyScope, values_per_result: Option) -> Self { let mut over = self.clone(); // Only a per-value statement extends to more values by the union diff --git a/crates/types/src/accuracy/mod.rs b/crates/types/src/accuracy/mod.rs new file mode 100644 index 000000000..0bcce29fb --- /dev/null +++ b/crates/types/src/accuracy/mod.rs @@ -0,0 +1,16 @@ +//! #511 §2.2: accuracy targets and the error metrics they are stated in +//! ([`AccuracyTarget`], [`AccuracyMetric`]), and machine-readable accuracy +//! guarantees ([`AccuracyGuarantee`]). +//! +//! - `target` — accuracy targets and the error metrics they are stated in. +//! - `guarantee` — machine-readable accuracy guarantees. +mod guarantee; +mod target; + +pub use guarantee::{ + AccuracyError, AccuracyGuarantee, BoundExpr, CompositionOperator, GuaranteeSource, + ProbabilityExpr, VALUES_PER_RESULT, +}; +pub use target::{ + default_metric_for_root, AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, NormScale, +}; diff --git a/crates/types/src/ir/properties/accuracy.rs b/crates/types/src/accuracy/target.rs similarity index 91% rename from crates/types/src/ir/properties/accuracy.rs rename to crates/types/src/accuracy/target.rs index 45f50b6b9..e460ae9af 100644 --- a/crates/types/src/ir/properties/accuracy.rs +++ b/crates/types/src/accuracy/target.rs @@ -3,7 +3,7 @@ //! A requirement and a guarantee share one outer shape, [`AccuracySpec`]: //! an error metric, an error ε, a failure probability δ and a scope. A //! requirement ([`AccuracyTarget`]) holds concrete numbers and may leave its -//! metric unset; a guarantee ([`crate::ir::properties::AccuracyGuarantee`]) +//! metric unset; a guarantee ([`crate::accuracy::AccuracyGuarantee`]) //! always states its metric and holds symbolic expressions that plan //! selection binds. A target without a metric accepts a guarantee in any //! metric that meets its ε, δ and scope. @@ -171,25 +171,10 @@ impl AccuracyScope { } } } - - /// Whether a statement over this scope also covers every value `other` - /// covers: the whole result covers every scope, a record share covers a - /// larger share (not every value: a smaller group is outside it), and a - /// single value only itself. - pub fn covers(self, other: Self) -> bool { - match (self, other) { - (Self::WholeResult, _) | (Self::PerValue, Self::PerValue) => true, - ( - Self::GroupsWithRecordShareAtLeast(share), - Self::GroupsWithRecordShareAtLeast(other_share), - ) => share <= other_share, - _ => false, - } - } } /// The outer shape of an accuracy guarantee -/// ([`crate::ir::properties::AccuracyGuarantee`]): +/// ([`crate::accuracy::AccuracyGuarantee`]): /// `Pr[err_metric(estimate, truth) > error] ≤ failure_probability` over the /// values `accuracy_scope` names. An [`AccuracyTarget`] has the same fields, with /// concrete numbers, an optional metric and `max_error` for `error`: a target @@ -318,9 +303,7 @@ pub fn default_metric_for_root(root: &QueryRoot) -> Option { /// intents without a listed default. fn intent_metric(intent: &AggIntent) -> Option { match intent { - AggIntent::Count { .. } | AggIntent::TopK { .. } => { - Some(AccuracyMetric::ABSOLUTE_STREAM_L1) - } + AggIntent::Count | AggIntent::TopK { .. } => Some(AccuracyMetric::ABSOLUTE_STREAM_L1), AggIntent::Cardinality { .. } | AggIntent::FrequencyL2 { .. } => { Some(AccuracyMetric::Relative) } @@ -384,7 +367,7 @@ fn node_metric(node: &OperatorNode) -> Option { /// select, order or rename rows, down to an aggregate's measure. A group /// key or any other expression is absolute. fn column_metric(node: &OperatorNode, column: usize) -> Option { - use crate::ir::operator::operator_properties::JoinKind; + use crate::ir::operator::arguments::JoinKind; let Operator::NonASAP(op) = &node.operator else { return Some(AccuracyMetric::ABSOLUTE); }; @@ -460,12 +443,12 @@ fn column_metric(node: &OperatorNode, column: usize) -> Option { /// count. fn aggregate_measure_at<'a>( node: &OperatorNode, - reduction: &crate::ir::operator::operator_properties::Reduction, + reduction: &crate::ir::operator::arguments::Reduction, measures: &'a [AggIntent], child: &OperatorNode, column: usize, ) -> Option<&'a AggIntent> { - use crate::ir::operator::operator_properties::Reduction; + use crate::ir::operator::arguments::Reduction; use crate::ir::scalar::{resolve_column_ref, ColumnRef}; let fields = &node.schema.fields; match reduction { @@ -538,11 +521,7 @@ mod tests { } fn quantile() -> AggIntent { - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::max_error(0.01), - } + AggIntent::Quantile { col: None, q: 0.99 } } /// A target reads and writes as a plain struct: every field but @@ -629,9 +608,7 @@ mod tests { }); let count = node(NonASAPOp::Aggregate { reduction: Reduction::PerEntity, - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::max_error(0.01), - }], + measures: vec![AggIntent::Count], output_names: vec![], filters: vec![], having: None, @@ -652,43 +629,25 @@ mod tests { #[test] fn root_metric_defaults_follow_the_query_intent() { let metric = |intent: AggIntent| default_metric_for_root(&aggregate(vec![intent])).unwrap(); - let exact = AccuracyTarget::exact(); - assert_eq!( - metric(AggIntent::Count { - accuracy: exact.clone() - }), - AccuracyMetric::ABSOLUTE_STREAM_L1 - ); + assert_eq!(metric(AggIntent::Count), AccuracyMetric::ABSOLUTE_STREAM_L1); assert_eq!( metric(AggIntent::Sum { col: None }), AccuracyMetric::ABSOLUTE ); assert_eq!( - metric(AggIntent::TopK { - k: 5, - accuracy: exact.clone() - }), + metric(AggIntent::TopK { k: 5 }), AccuracyMetric::ABSOLUTE_STREAM_L1 ); assert_eq!( - metric(AggIntent::Cardinality { - cols: vec![], - accuracy: exact.clone() - }), + metric(AggIntent::Cardinality { cols: vec![] }), AccuracyMetric::Relative ); assert_eq!( - metric(AggIntent::FrequencyL2 { - col: None, - accuracy: exact.clone() - }), + metric(AggIntent::FrequencyL2 { col: None }), AccuracyMetric::Relative ); assert_eq!( - metric(AggIntent::FrequencyEntropy { - col: None, - accuracy: exact.clone() - }), + metric(AggIntent::FrequencyEntropy { col: None }), AccuracyMetric::ABSOLUTE ); assert_eq!( @@ -708,18 +667,13 @@ mod tests { /// metrics, or any other expression, is absolute. #[test] fn root_metric_looks_through_a_null_guard_to_its_statistic() { - use crate::ir::operator::operator_properties::JoinKind; + use crate::ir::operator::arguments::JoinKind; use crate::ir::scalar::{CompareOpKind, ScalarValue}; use crate::ir::{Predicate, ProjectItem}; - let exact = AccuracyTarget::exact(); - let QueryRoot::Operator(l2) = aggregate(vec![AggIntent::FrequencyL2 { - col: None, - accuracy: exact.clone(), - }]) else { + let QueryRoot::Operator(l2) = aggregate(vec![AggIntent::FrequencyL2 { col: None }]) else { unreachable!() }; - let QueryRoot::Operator(count) = aggregate(vec![AggIntent::Count { accuracy: exact }]) - else { + let QueryRoot::Operator(count) = aggregate(vec![AggIntent::Count]) else { unreachable!() }; let joined = node(NonASAPOp::Join { @@ -847,22 +801,6 @@ mod tests { ); } - /// The whole result covers every scope; a record share covers a larger - /// share but not a single value, which may be a smaller group; a single - /// value covers only itself. - #[test] - fn a_wider_scope_covers_a_narrower_one() { - use AccuracyScope::*; - let share = GroupsWithRecordShareAtLeast; - assert!(WholeResult.covers(share(0.01))); - assert!(share(0.01).covers(share(0.05))); - assert!(!share(0.05).covers(PerValue)); - assert!(PerValue.covers(PerValue)); - assert!(!share(0.05).covers(share(0.01))); - assert!(!share(0.05).covers(WholeResult)); - assert!(!PerValue.covers(share(0.05))); - } - /// Exact means zero error with certainty, whatever the metric; an /// ε of zero without a δ is not exact. #[test] diff --git a/crates/logical-asap-optimization/src/deployment_capabilities.rs b/crates/types/src/deployment_capabilities.rs similarity index 70% rename from crates/logical-asap-optimization/src/deployment_capabilities.rs rename to crates/types/src/deployment_capabilities.rs index f449513b6..4c973a95a 100644 --- a/crates/logical-asap-optimization/src/deployment_capabilities.rs +++ b/crates/types/src/deployment_capabilities.rs @@ -3,27 +3,27 @@ //! //! [`DeploymentCapabilities`] says three things: //! -//! * which sketch families the executor supports, and which readouts it -//! supports for each. A family is supported when the executor can build +//! * which sketch summary types the executor supports, and which readouts +//! it supports for each. A summary type is supported when the executor can build //! its state, and a readout of it when the executor can answer that //! statistic ([`StatisticKind`]) from the state. Stage 1 (Pass 1) offers -//! no summary candidate whose family or readout is unsupported, so no +//! no summary candidate whose summary type or readout is unsupported, so no //! Pass 2 candidate has one either (Pass 2 rules share, split into tumbling windows -//! or reuse existing builds; none adds a family or a readout). Exact +//! or reuse existing builds; none adds a summary type or a readout). Exact //! accumulators and plain exact execution are always supported; //! * whether raw data samples are retained. If they are, Stage 3 counts them on //! disk once per source, at the longest window a query-time scan of it -//! reads. If they are not, raw data samples are not kept after they arrive, so +//! reads. If they are not, raw data samples are not retained after they arrive, so //! a candidate that scans raw data at query time is invalid (the //! rejection lists every such scan): only ingestion-time work reads raw //! rows. Stored data (at rest or mixed) with no raw data retained is a //! contradiction, refused as bad input ([`DeploymentCapabilities::validate`]); //! * whether query-time results are retained across evaluations. If they -//! are, Stage 2 also offers a tumbling-window chain whose readers evaluate -//! together at its tumbling-window width as `MatAtQueryKeptAcrossEvaluations`: each +//! are, Stage 2 also offers a tumbling-window chain whose queries evaluate +//! together at its tumbling-window width as query-time `AcrossEvaluations`: each //! evaluation builds one new tumbling window and keeps the others for the next ones, -//! covering the readers' longest look-back. A chain two or more queries -//! read keeps its `MatAtQuery` option too (shared within one evaluation), +//! covering the queries' longest look-back. A chain two or more queries +//! read keeps its `OneEvaluation` option too (shared within one evaluation), //! and Stage 3 picks the cheaper. If they are not, a query-time result is //! reused only within one evaluation. //! @@ -31,11 +31,11 @@ //! and keeps no query-time result across evaluations, which is the //! planner's behavior without deployment inputs. -// The executor lists the families it builds as `SupportedSummary`s, so the +// The executor lists the summary types it builds as `SupportedSummary`s, so the // type lives with the readout vocabulary in `asap-types`. -pub use asap_types::ir::schema::SupportedSummary; -use asap_types::ir::schema::{SketchAlgorithm, StatisticKind}; -use asap_types::workload::{DataArrival, DataWorkload}; +pub use crate::summary_type::SupportedSummary; +use crate::summary_type::{SketchAlgorithm, StatisticKind}; +use crate::workload::{DataArrival, DataWorkload}; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -46,14 +46,14 @@ use thiserror::Error; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct DeploymentCapabilities { - /// The sketch families the executor can build, each with the readouts - /// it can answer from one; `None` (the default) supports every family + /// The sketch summary types the executor can build, each with the + /// readouts it can answer from one; `None` (the default) supports every summary type /// and readout the planner can build. pub supported_summaries: Option>, - /// Whether raw data samples are kept for query-time scans; default `true`. + /// Whether raw data samples are retained for query-time scans; default `true`. pub raw_data_retained: bool, - /// Whether a query-time result may be kept across evaluations - /// (`MatAtQueryKeptAcrossEvaluations`); default `false`. + /// Whether a query-time result may be materialized across evaluations + /// (query-time `AcrossEvaluations`); default `false`. pub query_time_results_retained: bool, } @@ -68,9 +68,9 @@ impl Default for DeploymentCapabilities { } impl DeploymentCapabilities { - /// Whether the executor can build `family` and answer `readout` from it. - pub fn supports(&self, family: &SketchAlgorithm, readout: StatisticKind) -> bool { - summary_supported(self.supported_summaries.as_deref(), family, readout) + /// Whether the executor can build `summary_type` and answer `readout` from it. + pub fn supports(&self, summary_type: &SketchAlgorithm, readout: StatisticKind) -> bool { + summary_supported(self.supported_summaries.as_deref(), summary_type, readout) } /// Checks these capabilities against the data workload: stored data @@ -90,18 +90,18 @@ impl DeploymentCapabilities { } /// Whether an executor that builds `supported_summaries` (`None`: every -/// family and readout the planner can build) can build `family` and answer -/// `readout` from it. Pass 1 reads only this field of -/// [`DeploymentCapabilities`]. -pub(crate) fn summary_supported( +/// summary type and readout the planner can build) can build +/// `summary_type` and answer `readout` from it. Pass 1 reads only this +/// field of [`DeploymentCapabilities`]. +pub fn summary_supported( supported_summaries: Option<&[SupportedSummary]>, - family: &SketchAlgorithm, + summary_type: &SketchAlgorithm, readout: StatisticKind, ) -> bool { supported_summaries.is_none_or(|supported| { supported .iter() - .any(|s| s.family == *family && s.readouts.contains(&readout)) + .any(|s| s.summary_type == *summary_type && s.readouts.contains(&readout)) }) } @@ -122,7 +122,7 @@ pub enum DeploymentCapabilitiesError { mod tests { use super::*; - /// The default supports every family and readout, retains raw data and + /// The default supports every summary type and readout, retains raw data and /// keeps no query-time result across evaluations; a JSON object with /// only some fields keeps the defaults for the rest. #[test] @@ -131,7 +131,7 @@ mod tests { assert!(default.supports(&SketchAlgorithm::Bloom, StatisticKind::TopK)); assert!(default.raw_data_retained && !default.query_time_results_retained); let parsed: DeploymentCapabilities = serde_json::from_str( - r#"{"supported_summaries": [{"family": "Kll", "readouts": ["Quantile"]}]}"#, + r#"{"supported_summaries": [{"summary_type": "Kll", "readouts": ["Quantile"]}]}"#, ) .unwrap(); assert!(parsed.supports(&SketchAlgorithm::Kll, StatisticKind::Quantile)); diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index ac56e5b5f..e3f090294 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -13,12 +13,10 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; +use crate::accuracy::AccuracyGuarantee; use crate::ir::operator::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::query::QueryRoot; -use crate::ir::properties::execution_data_state::ExecutionTiming; -use crate::ir::properties::guarantee::AccuracyGuarantee; use crate::ir::schema::Schema; -use crate::physical::materialization::Materialization; /// Index of a node in [`FlatDag::nodes`]. pub type NodeId = usize; @@ -30,8 +28,6 @@ pub struct FlatNode { pub(crate) result_kind: OperatorResultKind, pub(crate) schema: Schema, pub(crate) guarantee: Option, - pub(crate) timing: Option, - pub(crate) materialization: Materialization, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -78,8 +74,6 @@ fn visit( result_kind: node.result_kind, schema: node.schema.clone(), guarantee: node.guarantee.clone(), - timing: node.timing, - materialization: node.materialization, }); originals.push(Rc::clone(node)); ids.insert(Rc::as_ptr(node), id); @@ -165,7 +159,6 @@ mod tests { #[test] fn json_round_trips() { let (dag, _) = flatten(&[QueryRoot::Operator(filter(values()))]); - assert!(dag.nodes.iter().all(|n| n.timing.is_none())); let json = serde_json::to_string(&dag).unwrap(); assert_eq!(serde_json::from_str::(&json).unwrap(), dag); } diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index 026dc620e..b8cf64393 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -12,11 +12,3 @@ pub use scalar::{ExprSemantics, Predicate, ProjectItem, ScalarExpr, SortKey}; pub use schema::error::SchemaDerivationError; pub mod flat; -pub mod materialized_view; -pub mod physical_export; -pub use properties::timing::{ - apply_materialization_timings, apply_materialization_timings_to_roots, data_state, - is_filtered_build, timing_under_consumer, MaterializationAssignment, NodeToTimedNodeCache, -}; -#[cfg(any(test, feature = "test-support"))] -pub use properties::timing::{planned_data_state, validate_at_ingestion}; diff --git a/crates/types/src/ir/operator/agg_intent.rs b/crates/types/src/ir/operator/agg_intent.rs index 4bde9c1b2..799ce1ade 100644 --- a/crates/types/src/ir/operator/agg_intent.rs +++ b/crates/types/src/ir/operator/agg_intent.rs @@ -1,14 +1,13 @@ //! The pre-ASAP aggregation-intent vocabulary — "what to compute, not how". //! -//! This IR carries intent ("compute a quantile to ε=0.01 accuracy"); the -//! choice between `HashAgg` / `SortAgg` / `SketchAgg(KLL{k=200})` is a +//! This IR carries intent ("compute a quantile"); an accuracy target belongs +//! to the query, not to a measure. The choice between `HashAgg` / `SortAgg` / `SketchAgg(KLL{k=200})` is a //! post-ASAP cost-aware decision, not encoded here. //! //! `AggIntent::TopK` is a first-class *intent*: "the k most frequent keys by -//! value, to accuracy ε." Like `Quantile`, the exact-vs-approximate -//! realisation — an exact heap / sort+limit when `accuracy: Exact`, a -//! heavy-hitter sketch when approximate — is a post-ASAP cost-aware decision, -//! not encoded here. The semantic distinction that *is* made at lowering is +//! value." Like `Quantile`, the exact-vs-approximate realisation — an exact +//! heap / sort+limit or a heavy-hitter sketch — is a post-ASAP cost-aware +//! decision, not encoded here. The semantic distinction that *is* made at lowering is //! intent vs operator: a heavy-hitter aggregate becomes `TopK`, whereas a //! generic `ORDER BY value LIMIT k` stays as the `NonASAPOp::Sort + Limit` //! operator pair. @@ -16,14 +15,13 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -use crate::ir::properties::AccuracyTarget; use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType}; /// "What to compute" — the vocabulary the planner pivots on. /// /// Grouping for `TopK` rides on the enclosing `NonASAPOp::Aggregate`'s `reduction` /// (positional `ColumnId`s), like every other aggregate; the intent itself -/// carries only `k` + the accuracy target. +/// carries only `k`. /// /// The single-column reducers (`Sum` / `Min` / `Max` / `Avg` / `StdDev` / /// `Variance` / `Quantile`) carry `col: Option` — the input @@ -45,9 +43,7 @@ use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType}; #[serde(bound(serialize = "C: Serialize", deserialize = "C: Deserialize<'de>"))] pub enum AggIntent { // ── Data-model-agnostic ────────────────────────────────────────────── - Count { - accuracy: AccuracyTarget, - }, + Count, Sum { col: Option, }, @@ -85,16 +81,14 @@ pub enum AggIntent { col: Option, #[serde(with = "crate::serde_f64")] q: f64, - accuracy: AccuracyTarget, }, - /// Heavy-hitter top-k to the given accuracy. The group-by keys live on + /// Heavy-hitter top-k. The group-by keys live on /// the enclosing `Aggregate.by`. // // Unlike `Quantile`/`Cardinality`, `TopK` ranks by the *aggregate output* // rather than a base column, so it carries no `col` — see #13 / #25. TopK { k: usize, - accuracy: AccuracyTarget, }, /// Distinct count over `cols`, in argument order. One column is SQL /// `COUNT(DISTINCT col)`; several count distinct *tuples* @@ -103,17 +97,14 @@ pub enum AggIntent { /// and `distinct_over_time` leave it so. Cardinality { cols: Vec, - accuracy: AccuracyTarget, }, /// L2 norm of the frequency vector of distinct input values. FrequencyL2 { col: Option, - accuracy: AccuracyTarget, }, /// Shannon entropy in bits of the frequency distribution of input values. FrequencyEntropy { col: Option, - accuracy: AccuracyTarget, }, // ── Time-series streaming derivatives ──────────────────────────────── @@ -491,7 +482,7 @@ impl AggIntent { /// locate it without an alias lookup. pub(crate) fn output_column(&self, input: &Field) -> Field { match self { - AggIntent::Count { .. } => col("count", DataType::Int64, false), + AggIntent::Count => col("count", DataType::Int64, false), AggIntent::Sum { .. } => col("sum", input.dtype.clone(), false), AggIntent::Min { .. } => col("min", input.dtype.clone(), input.nullable), AggIntent::Max { .. } => col("max", input.dtype.clone(), input.nullable), @@ -613,7 +604,7 @@ pub mod topk { impl Ranking { pub fn from_aggregate(aggregate: &AggIntent) -> Self { match aggregate { - AggIntent::Count { .. } => Self::Frequency, + AggIntent::Count => Self::Frequency, AggIntent::Sum { .. } => Self::WeightedSum, _ => Self::NonAdditive, } @@ -633,7 +624,7 @@ pub mod topk { pub fn agg_has_exact_per_entity_kernel(op: &AggIntent) -> bool { matches!( op, - AggIntent::Count { .. } + AggIntent::Count | AggIntent::Sum { .. } | AggIntent::Avg { .. } | AggIntent::Min { .. } @@ -648,19 +639,12 @@ pub fn agg_has_exact_per_entity_kernel(op: &AggIntent) -> bool { /// Default `Cardinality` intent over the sample value — HLL standard error at /// precision p=14. pub fn default_cardinality() -> AggIntent { - AggIntent::Cardinality { - cols: vec![], - accuracy: AccuracyTarget::max_error(1.04 / ((1u64 << 14) as f64).sqrt()), - } + AggIntent::Cardinality { cols: vec![] } } /// Default `Quantile` intent over the sample value at φ = `q`, `accuracy = ε 0.01`. pub fn default_quantile(q: f64) -> AggIntent { - AggIntent::Quantile { - col: None, - q, - accuracy: AccuracyTarget::max_error(0.01), - } + AggIntent::Quantile { col: None, q } } #[cfg(test)] @@ -689,10 +673,7 @@ mod tests { // output shape as the one-column form — the count of distinct tuples. #[test] fn distinct_tuple_cardinality_contract() { - let intent = AggIntent::Cardinality { - cols: vec![2, 5], - accuracy: AccuracyTarget::max_error(0.01), - }; + let intent = AggIntent::Cardinality { cols: vec![2, 5] }; assert_eq!(intent.input_cols(), vec![2, 5]); let output = intent.output_column(&c("x", DataType::Float64)); assert_eq!(output.name, "cardinality"); @@ -702,14 +683,7 @@ mod tests { #[test] fn output_column_names_are_intent_keyed() { let v = c("value", DataType::Float64); - assert_eq!( - AggIntent::::Count { - accuracy: AccuracyTarget::exact() - } - .output_column(&v) - .name, - "count" - ); + assert_eq!(AggIntent::::Count.output_column(&v).name, "count"); assert_eq!( AggIntent::::Sum { col: None } .output_column(&v) @@ -717,13 +691,9 @@ mod tests { "sum" ); assert_eq!( - AggIntent::::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::max_error(0.01) - } - .output_column(&v) - .name, + AggIntent::::Quantile { col: None, q: 0.99 } + .output_column(&v) + .name, "quantile_0_99" ); } @@ -752,12 +722,7 @@ mod tests { #[test] fn topk_ranking_classifies_aggregate_inputs() { use topk::Ranking::*; - assert_eq!( - topk::Ranking::from_aggregate(&AggIntent::Count { - accuracy: AccuracyTarget::exact() - }), - Frequency - ); + assert_eq!(topk::Ranking::from_aggregate(&AggIntent::Count), Frequency); assert_eq!( topk::Ranking::from_aggregate(&AggIntent::Sum { col: None }), WeightedSum @@ -785,38 +750,20 @@ mod tests { .is_empty(), "empty = PromQL sample value" ); - assert!(AggIntent::::Count { - accuracy: AccuracyTarget::exact() - } - .input_cols() - .is_empty()); + assert!(AggIntent::::Count.input_cols().is_empty()); } #[test] fn agg_intent_serde_roundtrip() { for v in [ - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::max_error(0.01), - }, + AggIntent::Quantile { col: None, q: 0.99 }, AggIntent::Quantile { col: Some(3), q: 0.99, - accuracy: AccuracyTarget::max_error(0.01), - }, - AggIntent::Cardinality { - cols: vec![2], - accuracy: AccuracyTarget::exact(), - }, - AggIntent::Cardinality { - cols: vec![2, 3], - accuracy: AccuracyTarget::exact(), - }, - AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::max_error(0.01), }, + AggIntent::Cardinality { cols: vec![2] }, + AggIntent::Cardinality { cols: vec![2, 3] }, + AggIntent::TopK { k: 10 }, ] { let json = serde_json::to_string(&v).unwrap(); let back: AggIntent = serde_json::from_str(&json).unwrap(); @@ -828,9 +775,9 @@ mod tests { #[test] fn cardinality_rejects_legacy_col_payloads() { for payload in [ - r#"{"kind":"cardinality","col":2,"accuracy":{"accuracy_metric":"absolute","max_error":0.0,"failure_probability":0.0,"accuracy_scope":"whole_result"}}"#, - r#"{"kind":"cardinality","col":null,"accuracy":{"accuracy_metric":"absolute","max_error":0.0,"failure_probability":0.0,"accuracy_scope":"whole_result"}}"#, - r#"{"kind":"cardinality","col":2,"cols":[1],"accuracy":{"accuracy_metric":"absolute","max_error":0.0,"failure_probability":0.0,"accuracy_scope":"whole_result"}}"#, + r#"{"kind":"cardinality","col":2}"#, + r#"{"kind":"cardinality","col":null}"#, + r#"{"kind":"cardinality","col":2,"cols":[1]}"#, ] { assert!( serde_json::from_str::(payload).is_err(), diff --git a/crates/types/src/ir/operator/operator_properties.rs b/crates/types/src/ir/operator/arguments.rs similarity index 100% rename from crates/types/src/ir/operator/operator_properties.rs rename to crates/types/src/ir/operator/arguments.rs diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 707ab4add..9b2dc957f 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -1,16 +1,19 @@ //! ASAP operators: summary-state construction, state operations and evaluations. -//! The summary family, kind/algorithm and parameters are committed here. +//! The summary type (family and algorithm) is an operator argument; θ is a +//! candidate's logical property. use std::rc::Rc; use serde::{Deserialize, Serialize}; -use super::node::{OperatorNode, OperatorResultKind}; +use super::input::{InputOrTemporaryBuild, OperatorInput}; +use super::node::{Operator, OperatorNode, OperatorResultKind}; +use crate::ir::operator::arguments::Reduction; use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; -use crate::ir::operator::operator_properties::Reduction; -use crate::ir::schema::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; -use crate::ir::schema::{ColumnId, DataType, FamilyCapability, Field, FieldDataType, Schema}; +use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; use crate::ir::SchemaDerivationError; +use crate::summary_type::SummaryTypeCapability; +use crate::summary_type::{SketchStatistic, SummaryUpdate}; /// Why an ASAP operator cannot be used yet. pub const UNIMPLEMENTED_ASAP_OP: &str = @@ -21,15 +24,14 @@ pub const UNIMPLEMENTED_ASAP_OP: &str = #[serde(bound(deserialize = "C: Deserialize<'de>"))] pub enum ASAPOp> { /// Summary aggregation. Output: grouping columns + one field carrying - /// partial summary state per group, typed `family`. + /// partial summary state per group, typed `summary_type`. SummaryAgg { child: C, - /// Which summary family realizes this aggregation. Never + /// The summary type (family and algorithm) of the state. Never /// `FieldDataType::Plain`. - family: FieldDataType, + summary_type: FieldDataType, input: SummaryUpdate, reduction: Reduction, - grouping: GroupingStrategy, #[serde(default)] filter: Option>, }, @@ -55,21 +57,12 @@ pub enum ASAPOp> { evaluation: PopulationStatistic, }, /// Merge compatible partial states of one family: every input state of - /// a group, including each tumbling window of a [`SummaryTumblingWindowChain`](ASAPOp::SummaryTumblingWindowChain), - /// into one state per group. + /// a group into one state per group, or, with `group_by`, regrouped. + /// Over a time-indexed state (a `TimeRange` over a + /// [`SummaryTumblingWindowChain`](ASAPOp::SummaryTumblingWindowChain)), the windows it + /// selects; there is no window-selection field. SummaryMerge { children: Vec, - /// Merge only the newest `n` tumbling windows of the one `SummaryTumblingWindowChain` this - /// merge reads, `1 ≤ n <` its `tumbling_window_count`: the state of one build - /// over the newest `n · tumbling_window_width_ms` of the chain's window, so a - /// query over a shorter window reads a chain kept for a longer one. - /// `None` merges every input state. - #[serde( - default, - skip_serializing_if = "Option::is_none", - alias = "newest_panes" - )] - newest_tumbling_windows: Option, /// A rollup (regrouping merge): the input's plain columns the /// merged states are keyed by, in output order. Every input state /// whose values in these columns match goes into one output state, @@ -78,33 +71,26 @@ pub enum ASAPOp> { #[serde(default, skip_serializing_if = "Option::is_none")] group_by: Option>, }, - /// A tumbling-window summary (#509 Pass 2, window-composition - /// rule): the `SummaryAgg` with these `family`, `input`, `reduction`, - /// `grouping` and `filter`, built over each of `tumbling_window_count` back-to-back - /// range windows of `tumbling_window_width_ms` that end at `child`'s evaluation - /// time, newest first. `child` is the input a range selector would - /// read, so tumbling window `i` covers `(-(i+1)·w, -i·w]` relative to it. Output: - /// the `SummaryAgg` schema, one state per group and tumbling window; a - /// `SummaryMerge` over the chain gives the state of one build over the - /// whole `tumbling_window_count · tumbling_window_width_ms` window - /// (`whole_window_build`). - /// One compact node whatever the tumbling-window count: costs and guarantees come - /// from formulas, and the executor expands the tumbling windows - /// (`expand_tumbling_window_chains`). + /// A chain of tumbling windows (concepts doc, section 3, "Chain + /// groups"): the `SummaryAgg` with these `summary_type`, `input`, + /// `reduction` and `filter`, built over each window + /// `(k·w, (k+1)·w]` of `child`, `w` = `tumbling_window_width_ms`, on a + /// grid anchored at the epoch. Each window is built when it ends. Output: + /// one row per window, (window end, keys, state), a state with a time + /// index. It has no range and no window count: a `TimeRange` (and + /// `TimeShift`) over it selects the windows a query reads, and how many + /// windows it retains is $h_v$, a Stage 3 choice. // Written as `PaneChain` before #1163; the aliases read those documents. #[serde(alias = "PaneChain")] SummaryTumblingWindowChain { child: C, - family: FieldDataType, + summary_type: FieldDataType, input: SummaryUpdate, reduction: Reduction, - grouping: GroupingStrategy, #[serde(default)] filter: Option>, #[serde(alias = "pane_width_ms")] tumbling_window_width_ms: u64, - #[serde(alias = "pane_count")] - tumbling_window_count: u32, }, // ── Reserved: migrated but unimplemented ── SummarySubtract { @@ -119,7 +105,7 @@ pub enum ASAPOp> { outer: C, inner: C, key: ColumnId, - family: FieldDataType, + summary_type: FieldDataType, }, Extension { child: C, @@ -158,41 +144,35 @@ impl ASAPOp { match self { SummaryAgg { child, - family, + summary_type: family, input, reduction, - grouping, filter, } => SummaryAgg { child: f(child), - family: family.clone(), + summary_type: family.clone(), input: input.clone(), reduction: reduction.clone(), - grouping: grouping.clone(), filter: filter .as_ref() .map(|p| crate::ir::scalar::Predicate(p.0.map_operator_refs(&mut f))), }, SummaryTumblingWindowChain { child, - family, + summary_type: family, input, reduction, - grouping, filter, tumbling_window_width_ms, - tumbling_window_count, } => SummaryTumblingWindowChain { child: f(child), - family: family.clone(), + summary_type: family.clone(), input: input.clone(), reduction: reduction.clone(), - grouping: grouping.clone(), filter: filter .as_ref() .map(|p| crate::ir::scalar::Predicate(p.0.map_operator_refs(&mut f))), tumbling_window_width_ms: *tumbling_window_width_ms, - tumbling_window_count: *tumbling_window_count, }, SummaryEstimate { summary_input, @@ -210,13 +190,8 @@ impl ASAPOp { child: f(child), evaluation: evaluation.clone(), }, - SummaryMerge { - children, - newest_tumbling_windows, - group_by, - } => SummaryMerge { + SummaryMerge { children, group_by } => SummaryMerge { children: children.iter().map(&mut f).collect(), - newest_tumbling_windows: *newest_tumbling_windows, group_by: group_by.clone(), }, SummarySubtract { left, right } => SummarySubtract { @@ -231,12 +206,12 @@ impl ASAPOp { outer, inner, key, - family, + summary_type: family, } => SummaryJoin { outer: f(outer), inner: f(inner), key: *key, - family: family.clone(), + summary_type: family.clone(), }, Extension { child, name } => Extension { child: f(child), @@ -263,7 +238,7 @@ impl ASAPOp { } } -impl ASAPOp { +impl ASAPOp { fn unimplemented() -> SchemaDerivationError { SchemaDerivationError::InvalidScalarSignature(UNIMPLEMENTED_ASAP_OP.into()) } @@ -271,12 +246,21 @@ impl ASAPOp { /// The summary state this operator produces, if it produces state. pub fn produced_state(&self) -> Option<&FieldDataType> { match self { - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. } - | ASAPOp::SummaryJoin { family, .. } => Some(family), + ASAPOp::SummaryAgg { + summary_type: family, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + summary_type: family, + .. + } + | ASAPOp::SummaryJoin { + summary_type: family, + .. + } => Some(family), ASAPOp::SummaryMerge { children, .. } => children.first().and_then(|child| { child - .schema + .schema() .fields .iter() .find(|field| !field.is_plain()) @@ -295,38 +279,21 @@ impl ASAPOp { Ok(match self { SummaryAgg { child, - family, + summary_type: family, reduction, .. } | SummaryTumblingWindowChain { child, - family, + summary_type: family, reduction, .. - } => { - let mut schema = crate::ir::schema::aggregate_schema::aggregate_output_schema( - &child.schema, - reduction, - &[crate::ir::operator::AggIntent::Sum { col: None }], - &[], - )?; - let index = match reduction { - Reduction::PerEntity => crate::ir::scalar::resolve_column_ref( - &crate::ir::scalar::ColumnRef::SampleValue, - &schema, - ) - .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e.to_string()))?, - Reduction::Reduce(_) => schema.fields.len() - 1, - }; - schema.fields[index] = Field::new("state", family.clone(), false); - schema - } + } => summary_agg_schema(child.schema(), family, reduction)?, SummaryEstimate { summary_input, query, } => { - let input = &summary_input.schema; + let input = summary_input.schema(); let (name, mut dtype) = match query { SketchStatistic::Quantile { .. } => ("quantile", DataType::Float64), SketchStatistic::Cardinality => ("cardinality", DataType::Int64), @@ -336,7 +303,7 @@ impl ASAPOp { SketchStatistic::TopK { .. } => ("topk", DataType::Utf8), }; if matches!( - summary_input.summary_build(), + summary_build(summary_input), Some( SummaryAgg { reduction: Reduction::PerEntity, @@ -367,107 +334,29 @@ impl ASAPOp { } FinalizeExactAccumulator { child } => { // A rollup finalizes as the coarse build it stands for. - if let Some(build) = rolled_up_build(child) { - return FinalizeExactAccumulator { child: build }.output_schema(); + if let Some((build, schema)) = rolled_up_build(child) { + return finalized_schema(Some(&build), &schema); } - // Merged tumbling windows finalize as one build over their window. if super::tumbling_window_chain::merged_chain(child).is_some() { - return FinalizeExactAccumulator { - child: Rc::new(super::tumbling_window_chain::merged_window_build(child)?), - } - .output_schema(); + todo!("#1207 PR 1: finalize the windows a TimeRange over a chain selects as one build over that TimeRange") } - let value_result = if let Some(ASAPOp::SummaryAgg { - child: source, - family: FieldDataType::ExactAggregate(kind, _), - input, - reduction, - filter, - .. - }) = child.summary_build() - { - { - use crate::ir::operator::AggIntent; - use crate::ir::schema::{ExactKind, SummaryInputExpr}; - let column = match &input.weight { - SummaryInputExpr::Column(col) => Some( - crate::ir::scalar::column_resolution::resolve_column_ref( - col, - &source.schema, - ) - .map_err(|e| { - SchemaDerivationError::InvalidScalarSignature(e.to_string()) - })?, - ), - _ => None, - }; - let measure = match kind { - ExactKind::Sum => Some(AggIntent::Sum { col: column }), - ExactKind::Min => Some(AggIntent::Min { col: column }), - ExactKind::Max => Some(AggIntent::Max { col: column }), - ExactKind::Count => Some(AggIntent::Count { - accuracy: crate::ir::properties::AccuracyTarget::exact(), - }), - ExactKind::Rate => Some(AggIntent::Rate), - ExactKind::IRate => Some(AggIntent::IRate), - ExactKind::Increase => Some(AggIntent::Increase), - }; - measure - .map(|measure| { - crate::ir::NonASAPOp::Aggregate { - child: Rc::clone(source), - reduction: reduction.clone(), - measures: vec![measure], - output_names: vec![], - // A filtered SUM/MIN/MAX is NULL for a - // group without a matching row. - filters: vec![filter.clone()], - having: None, - } - .output_schema() - }) - .transpose()? - .and_then(|s| match reduction { - Reduction::PerEntity => { - s.column_id("value").and_then(|i| s.fields.get(i).cloned()) - } - Reduction::Reduce(_) => s.fields.last().cloned(), - }) - } - } else { - None - }; - let mut out = child.schema.clone(); - for f in &mut out.fields { - if let FieldDataType::ExactAggregate(kind, _) = &f.dtype { - if let Some(result) = &value_result { - // The finalized value replaces the aggregate it - // realizes, so it takes that aggregate's column. - f.name = result.name.clone(); - f.dtype = result.dtype.clone(); - f.nullable = result.nullable; - } else { - f.dtype = FieldDataType::Plain(finalized_data_type(kind)); - } - } - } - out + finalized_schema(summary_build(child).as_ref(), child.schema())? } - MaintainPopulation { child, .. } => child.schema.clone(), + MaintainPopulation { child, .. } => child.schema().clone(), EvaluatePopulation { child, evaluation } => { use crate::ir::operator::maintained_population::PopulationInput; use crate::ir::operator::{AggIntent, GroupKeys}; - let Some(MaintainPopulation { + let Operator::ASAP(MaintainPopulation { child: source, population, - }) = child.asap() + }) = child.operator() else { return Err(SchemaDerivationError::InvalidScalarSignature( "population evaluation requires maintained membership".into(), )); }; if matches!(evaluation, PopulationStatistic::TopK { .. }) { - source.schema.clone() + source.schema().clone() } else { let (keys, column) = match &population.input { PopulationInput::Rows { @@ -480,7 +369,7 @@ impl ASAPOp { .grouping .iter() .map(|name| { - source.schema.column_id(name).ok_or_else(|| { + source.schema().column_id(name).ok_or_else(|| { SchemaDerivationError::InvalidScalarSignature( "population grouping column is absent".into(), ) @@ -497,15 +386,12 @@ impl ASAPOp { ) } }; - let accuracy = crate::ir::properties::AccuracyTarget::exact(); let measure = match evaluation { - PopulationStatistic::Quantile { q } => AggIntent::Quantile { - q: *q, - col: column, - accuracy, - }, + PopulationStatistic::Quantile { q } => { + AggIntent::Quantile { q: *q, col: column } + } PopulationStatistic::Sum => AggIntent::Sum { col: column }, - PopulationStatistic::Count => AggIntent::Count { accuracy }, + PopulationStatistic::Count => AggIntent::Count, PopulationStatistic::Average => AggIntent::Avg { col: column }, PopulationStatistic::TopK { .. } => unreachable!(), }; @@ -524,7 +410,7 @@ impl ASAPOp { children, group_by, .. } => { self.validate_inputs()?; - let input = &children[0].schema; + let input = children[0].schema(); match group_by { None => input.clone(), Some(keys) => regrouped_schema(input, keys)?, @@ -558,46 +444,18 @@ impl ASAPOp { /// Local input-contract checks. pub(crate) fn validate_inputs(&self) -> Result<(), SchemaDerivationError> { use ASAPOp::*; - let needs_state = |node: &OperatorNode, what: &str| { - if node.result_kind != OperatorResultKind::State { + let needs_state = |node: &I, what: &str| { + if node.result_kind() != OperatorResultKind::State { Err(SchemaDerivationError::InvalidScalarSignature(format!( "{what} requires summary state as input, got {:?}", - node.result_kind + node.result_kind() ))) } else { Ok(()) } }; match self { - SummaryMerge { - children, - newest_tumbling_windows, - group_by, - } => { - if newest_tumbling_windows.is_some() && group_by.is_some() { - return Err(SchemaDerivationError::InvalidScalarSignature( - "a rollup merges every input state, not the newest tumbling windows".into(), - )); - } - if let Some(n) = newest_tumbling_windows { - let tumbling_window_count = match children.as_slice() { - [chain] => match chain.asap() { - Some(SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => Some(*tumbling_window_count), - _ => None, - }, - _ => None, - }; - if !tumbling_window_count.is_some_and(|count| (1..count).contains(n)) { - return Err(SchemaDerivationError::InvalidScalarSignature( - "a merge of the newest tumbling windows reads one tumbling-window chain, and fewer \ - tumbling windows than it has" - .into(), - )); - } - } + SummaryMerge { children, .. } => { let Some(first) = children.first() else { return Err(SchemaDerivationError::InvalidScalarSignature( "summary merge requires at least one state input".into(), @@ -606,7 +464,7 @@ impl ASAPOp { // Matching state parameters and grouping positions are necessary; // matching names alone cannot prove two states compatible. if first - .schema + .schema() .fields .iter() .filter(|field| !field.is_plain()) @@ -619,7 +477,7 @@ impl ASAPOp { } for child in children { needs_state(child, "SummaryMerge")?; - if child.schema != first.schema { + if child.schema() != first.schema() { return Err(SchemaDerivationError::InvalidScalarSignature( "summary merge inputs must have identical state and grouping schemas" .into(), @@ -636,18 +494,18 @@ impl ASAPOp { query, } => { needs_state(summary_input, "SummaryEstimate")?; - // The family-capability table is the one source of which - // statistic a state's family answers. + // The capability table is the one source of which statistic + // a summary type answers. let states: Vec<_> = summary_input - .schema + .schema() .fields .iter() .filter(|f| !f.is_plain()) .collect(); let valid = match states.as_slice() { [field] => match &field.dtype { - family @ FieldDataType::Sketch(..) => { - FamilyCapability::of(family).answers(query) + summary_type @ FieldDataType::Sketch(..) => { + SummaryTypeCapability::of(summary_type).answers(query) } _ => false, }, @@ -655,7 +513,7 @@ impl ASAPOp { }; if !valid { return Err(SchemaDerivationError::InvalidScalarSignature( - "evaluation does not match its summary family".into(), + "evaluation does not match its summary type".into(), )); } Ok(()) @@ -663,7 +521,7 @@ impl ASAPOp { FinalizeExactAccumulator { child } => { needs_state(child, "FinalizeExactAccumulator")?; if child - .schema + .schema() .fields .iter() .all(|f| !matches!(f.dtype, FieldDataType::ExactAggregate(..))) @@ -676,7 +534,7 @@ impl ASAPOp { } EvaluatePopulation { child, evaluation } => { needs_state(child, "EvaluatePopulation")?; - if !matches!(child.asap(), Some(MaintainPopulation { population, .. }) if population.supports(evaluation)) + if !matches!(child.operator(), Operator::ASAP(MaintainPopulation { population, .. }) if population.supports(evaluation)) { return Err(SchemaDerivationError::InvalidScalarSignature( "population evaluation requires compatible maintained membership".into(), @@ -687,23 +545,22 @@ impl ASAPOp { SummaryTumblingWindowChain { child, tumbling_window_width_ms, - tumbling_window_count, .. } => { - if *tumbling_window_width_ms == 0 || *tumbling_window_count == 0 { + if *tumbling_window_width_ms == 0 { return Err(SchemaDerivationError::InvalidScalarSignature( - "a tumbling-window chain needs a positive tumbling-window width and tumbling-window count".into(), + "a tumbling-window chain needs a positive tumbling-window width".into(), )); } // The chain is the range selector: it reads the series the // selector would, or a SQL table's rows for a trailing window. if !matches!( - child.result_kind, + child.result_kind(), OperatorResultKind::InstantVector | OperatorResultKind::Relation ) { return Err(SchemaDerivationError::InvalidScalarSignature(format!( "a tumbling-window chain reads an instant vector or a relation, got {:?}", - child.result_kind + child.result_kind() ))); } self.summary_agg_inputs() @@ -725,14 +582,14 @@ impl ASAPOp { fn summary_agg_inputs(&self) -> Result<(), SchemaDerivationError> { let (ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, filter, .. } | ASAPOp::SummaryTumblingWindowChain { child, - family, + summary_type: family, input, filter, .. @@ -740,16 +597,16 @@ impl ASAPOp { else { unreachable!("called on a summary build"); }; - if family.is_plain() || child.result_kind == OperatorResultKind::State { + if family.is_plain() || child.result_kind() == OperatorResultKind::State { return Err(SchemaDerivationError::InvalidScalarSignature( "summary aggregation requires values and produces a state family".into(), )); } fn check( - expr: &crate::ir::schema::SummaryInputExpr, + expr: &crate::summary_type::SummaryInputExpr, schema: &Schema, ) -> Result<(), SchemaDerivationError> { - use crate::ir::schema::SummaryInputExpr; + use crate::summary_type::SummaryInputExpr; match expr { SummaryInputExpr::Column(col) => { crate::ir::scalar::resolve_column_ref(col, schema).map_err(|e| { @@ -765,12 +622,12 @@ impl ASAPOp { } Ok(()) } - check(&input.weight, &child.schema)?; + check(&input.weight, child.schema())?; if let Some(item) = &input.item { - check(item, &child.schema)?; + check(item, child.schema())?; } if let Some(filter) = filter { - if filter.0.scalar_type(&child.schema)?.0 != DataType::Bool { + if filter.0.scalar_type(child.schema())?.0 != DataType::Bool { return Err(SchemaDerivationError::InvalidScalarSignature( "summary filter must be boolean".into(), )); @@ -781,57 +638,168 @@ impl ASAPOp { } /// The build a rollup (a `SummaryMerge` with `group_by` of one input) -/// stands for: the input's build (one build over a merged chain's window) -/// grouped by the input columns the rollup's keys name. `None` for any -/// other node, or a rollup whose build does not derive. -fn rolled_up_build(state: &OperatorNode) -> Option> { - let ASAPOp::SummaryMerge { +/// stands for, with its schema: the input's build (one build over a merged +/// chain's window) grouped by the input columns the rollup's keys name. +/// `None` for any other input, or a rollup whose build does not derive. +fn rolled_up_build( + state: &I, +) -> Option<(ASAPOp>, Schema)> { + let Operator::ASAP(ASAPOp::SummaryMerge { children, group_by: Some(keys), .. - } = state.asap()? + }) = state.operator() else { return None; }; let [input] = children.as_slice() else { return None; }; - let input = selected_from(input).unwrap_or(input); - let build = match input.asap()? { - ASAPOp::SummaryAgg { .. } => (**input).clone(), - _ if super::tumbling_window_chain::merged_chain(input).is_some() => { - super::tumbling_window_chain::merged_window_build(input).ok()? + let input = selected_from(input).unwrap_or_else(|| input.clone()); + let build = match input.operator() { + Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => { + op.map_children(|child| InputOrTemporaryBuild::Input(child.clone())) + } + _ if super::tumbling_window_chain::merged_chain(&input).is_some() => { + todo!("#1207 PR 1: roll up the windows a TimeRange over a chain selects as one build over that TimeRange") } _ => return None, }; let ASAPOp::SummaryAgg { child, - family, + summary_type: family, input: update, reduction, - grouping, filter, - } = build.asap()? + } = build else { return None; }; let mapped = keys .iter() - .map(|&key| match reduction { + .map(|&key| match &reduction { Reduction::Reduce(by) if !by.is_without() => by.keys().get(key).copied(), Reduction::PerEntity => Some(key), Reduction::Reduce(_) => None, }) .collect::>>()?; - OperatorNode::new_shared(super::node::Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: family.clone(), - input: update.clone(), - reduction: Reduction::by(mapped), - grouping: grouping.clone(), - filter: filter.clone(), - })) - .ok() + let reduction = Reduction::by(mapped); + let schema = summary_agg_schema(child.schema(), &family, &reduction).ok()?; + Some(( + ASAPOp::SummaryAgg { + child, + summary_type: family, + input: update, + reduction, + filter, + }, + schema, + )) +} + +/// The schema of a `SummaryAgg` (or `SummaryTumblingWindowChain`) of +/// `summary_type` by `reduction` over rows of schema `child`: the reduction's +/// output columns, its value column replaced by the state. +pub(crate) fn summary_agg_schema( + child: &Schema, + summary_type: &FieldDataType, + reduction: &Reduction, +) -> Result { + let mut schema = crate::ir::schema::aggregate_schema::aggregate_output_schema( + child, + reduction, + &[crate::ir::operator::AggIntent::Sum { col: None }], + &[], + )?; + let index = match reduction { + Reduction::PerEntity => crate::ir::scalar::resolve_column_ref( + &crate::ir::scalar::ColumnRef::SampleValue, + &schema, + ) + .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e.to_string()))?, + Reduction::Reduce(_) => schema.fields.len() - 1, + }; + schema.fields[index] = Field::new("state", summary_type.clone(), false); + Ok(schema) +} + +/// The schema `FinalizeExactAccumulator` derives over state of +/// `state_schema` that `build` (the state's `SummaryAgg`, if any) builds: +/// each exact aggregate replaced by the value it finalizes to. +fn finalized_schema( + build: Option<&ASAPOp>, + state_schema: &Schema, +) -> Result { + let value_result = if let Some(ASAPOp::SummaryAgg { + child: source, + summary_type: FieldDataType::ExactAggregate(kind, _), + input, + reduction, + filter, + .. + }) = build + { + { + use crate::ir::operator::AggIntent; + use crate::summary_type::{ExactKind, SummaryInputExpr}; + let column = match &input.weight { + SummaryInputExpr::Column(col) => Some( + crate::ir::scalar::column_resolution::resolve_column_ref(col, source.schema()) + .map_err(|e| { + SchemaDerivationError::InvalidScalarSignature(e.to_string()) + })?, + ), + _ => None, + }; + let measure = match kind { + ExactKind::Sum => Some(AggIntent::Sum { col: column }), + ExactKind::Min => Some(AggIntent::Min { col: column }), + ExactKind::Max => Some(AggIntent::Max { col: column }), + ExactKind::Count => Some(AggIntent::Count), + ExactKind::Rate => Some(AggIntent::Rate), + ExactKind::IRate => Some(AggIntent::IRate), + ExactKind::Increase => Some(AggIntent::Increase), + }; + measure + .map(|measure| { + crate::ir::NonASAPOp::Aggregate { + child: source.clone(), + reduction: reduction.clone(), + measures: vec![measure], + output_names: vec![], + // A filtered SUM/MIN/MAX is NULL for a + // group without a matching row. + filters: vec![filter.clone()], + having: None, + } + .output_schema() + }) + .transpose()? + .and_then(|s| match reduction { + Reduction::PerEntity => { + s.column_id("value").and_then(|i| s.fields.get(i).cloned()) + } + Reduction::Reduce(_) => s.fields.last().cloned(), + }) + } + } else { + None + }; + let mut out = state_schema.clone(); + for f in &mut out.fields { + if let FieldDataType::ExactAggregate(kind, _) = &f.dtype { + if let Some(result) = &value_result { + // The finalized value replaces the aggregate it + // realizes, so it takes that aggregate's column. + f.name = result.name.clone(); + f.dtype = result.dtype.clone(); + f.nullable = result.nullable; + } else { + f.dtype = FieldDataType::Plain(finalized_data_type(kind)); + } + } + } + Ok(out) } /// The schema of a regrouping `SummaryMerge` over states of schema `input`: @@ -867,8 +835,8 @@ fn regrouped_schema(input: &Schema, keys: &[ColumnId]) -> Result DataType { - use crate::ir::schema::sketch::ExactKind; +fn finalized_data_type(kind: &crate::summary_type::ExactKind) -> DataType { + use crate::summary_type::ExactKind; match kind { ExactKind::Count => DataType::Int64, _ => DataType::Float64, @@ -877,11 +845,11 @@ fn finalized_data_type(kind: &crate::ir::schema::sketch::ExactKind) -> DataType /// The states `node` selects from when it is a `Filter` that keeps whole /// summary states by their plain columns (its output is state); `None` for -/// any other node. -pub fn selected_from(node: &OperatorNode) -> Option<&Rc> { - match node.non_asap() { - Some(crate::ir::NonASAPOp::Filter { child, .. }) - if node.result_kind == OperatorResultKind::State => +/// any other input. +pub fn selected_from(node: &I) -> Option { + match node.operator() { + Operator::NonASAP(crate::ir::NonASAPOp::Filter { child, .. }) + if node.result_kind() == OperatorResultKind::State => { Some(child) } @@ -889,18 +857,38 @@ pub fn selected_from(node: &OperatorNode) -> Option<&Rc> { } } +/// The build whose state this summary state carries: a `SummaryAgg` or +/// `SummaryTumblingWindowChain` itself, or the first input's build under +/// `SummaryMerge`s (merged inputs have one definition, which summary +/// coverage checks). `None` under a regrouping merge, whose states no build +/// carries (its coverage's definition is the build it stands for), and for +/// any other input. +pub fn summary_build(state: &I) -> Option> { + match state.operator() { + Operator::ASAP( + op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), + ) => Some(op), + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + .. + }) => summary_build(children.first()?), + _ => None, + } +} + /// The category of the values a evaluation of `state` produces: the category /// of the relational input the state was built from. -fn source_kind(node: &OperatorNode) -> OperatorResultKind { +fn source_kind(node: &I) -> OperatorResultKind { if let Some(states) = selected_from(node) { - return source_kind(states); + return source_kind(&states); } - match &node.operator { - super::node::Operator::ASAP(op) => match op.children().first() { - Some(child) => source_kind(child), + match node.operator() { + Operator::ASAP(op) => match op.children().first() { + Some(child) => source_kind(*child), None => OperatorResultKind::Relation, }, - super::node::Operator::NonASAP(_) => match node.result_kind { + Operator::NonASAP(_) => match node.result_kind() { OperatorResultKind::RangeVector => OperatorResultKind::InstantVector, OperatorResultKind::State => OperatorResultKind::Relation, other => other, @@ -914,21 +902,28 @@ fn source_kind(node: &OperatorNode) -> OperatorResultKind { /// series identity), then the estimated weight, `topk` (Float64). `None` /// when `state` is not a keyed build whose item columns resolve in its /// input. -fn topk_estimate_schema(state: &OperatorNode) -> Option { - use crate::ir::schema::SummaryInputExpr; +fn topk_estimate_schema(state: &I) -> Option { // A rollup reads items as the coarse build it stands for. - let rolled_up = rolled_up_build(state); - let build = match &rolled_up { - Some(build) => build.asap(), - None => state.summary_build(), - }; - let (Some(ASAPOp::SummaryAgg { child, input, .. }) - | Some(ASAPOp::SummaryTumblingWindowChain { child, input, .. })) = build + match rolled_up_build(state) { + Some((build, _)) => topk_schema_of_build(&build, state.schema()), + None => topk_schema_of_build(&summary_build(state)?, state.schema()), + } +} + +/// [`topk_estimate_schema`] of state of `state_schema` built by `build`. +fn topk_schema_of_build( + build: &ASAPOp, + state_schema: &Schema, +) -> Option { + use crate::summary_type::SummaryInputExpr; + let (ASAPOp::SummaryAgg { child, input, .. } + | ASAPOp::SummaryTumblingWindowChain { child, input, .. }) = build else { return None; }; + let child_schema = child.schema(); let resolve = |column: &crate::ir::scalar::ColumnRef| { - crate::ir::scalar::resolve_column_ref(column, &child.schema).ok() + crate::ir::scalar::resolve_column_ref(column, child_schema).ok() }; let columns = match input.item.as_ref()? { SummaryInputExpr::Column(column) => vec![resolve(column)?], @@ -939,14 +934,14 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { _ => None, }) .collect::>>()?, - SummaryInputExpr::EntityIdentity(crate::ir::schema::EntityIdentity::PromqlLabelSet { + SummaryInputExpr::EntityIdentity(crate::summary_type::EntityIdentity::PromqlLabelSet { excluding, }) => { let excluded: Vec<_> = excluding.iter().filter_map(resolve).collect(); let value = resolve(&crate::ir::scalar::ColumnRef::SampleValue); - (0..child.schema.fields.len()) + (0..child_schema.fields.len()) .filter(|index| { - Some(*index) != child.schema.time_index + Some(*index) != child_schema.time_index && Some(*index) != value && !excluded.contains(index) }) @@ -960,12 +955,11 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { let complete_item_identity = matches!( input.item.as_ref()?, SummaryInputExpr::Column(_) | SummaryInputExpr::Tuple(_) - ) || child.schema.closed + ) || child_schema.closed || columns.iter().any(|&index| { - child.schema.fields[index].name == crate::ir::schema::PROMQL_SERIES_IDENTITY + child_schema.fields[index].name == crate::ir::schema::PROMQL_SERIES_IDENTITY }); - let mut fields: Vec = state - .schema + let mut fields: Vec = state_schema .fields .iter() .filter(|field| field.is_plain()) @@ -974,9 +968,9 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { for index in columns { if !fields .iter() - .any(|field| field.name == child.schema.fields[index].name) + .any(|field| field.name == child_schema.fields[index].name) { - fields.push(child.schema.fields[index].clone()); + fields.push(child_schema.fields[index].clone()); } } // One row per (partition, item), not one row per partition. Preserve this @@ -990,7 +984,7 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { Some(Schema { fields, unique_keys, - ..state.schema.clone() + ..state_schema.clone() }) } @@ -999,18 +993,18 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { /// through `SummaryMerge`) is filtered, or, over a relation, is global (zero /// rows give one empty state) or summarizes a nullable column (a group of /// NULLs gives an empty state). -fn quantile_may_be_empty(state: &OperatorNode) -> bool { - match state.asap() { - Some(ASAPOp::SummaryMerge { +fn quantile_may_be_empty(state: &I) -> bool { + match state.operator() { + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { // A global rollup must also represent the empty input population, // even when every existing grouped state is nonempty. (source_kind(state) == OperatorResultKind::Relation && group_by.as_ref().is_some_and(Vec::is_empty)) - || children.iter().any(|child| quantile_may_be_empty(child)) + || children.iter().any(quantile_may_be_empty) } - Some( + Operator::ASAP( ASAPOp::SummaryAgg { child, input, @@ -1029,15 +1023,15 @@ fn quantile_may_be_empty(state: &OperatorNode) -> bool { if filter.is_some() { return true; } - if child.result_kind != OperatorResultKind::Relation { + if child.result_kind() != OperatorResultKind::Relation { return false; } let global = matches!(reduction, Reduction::Reduce(by) if by.keys().is_empty()); let nullable = match &input.weight { - crate::ir::schema::sketch::SummaryInputExpr::Column(column) => { - crate::ir::scalar::resolve_column_ref(column, &child.schema) + crate::summary_type::SummaryInputExpr::Column(column) => { + crate::ir::scalar::resolve_column_ref(column, child.schema()) .ok() - .and_then(|index| child.schema.fields.get(index)) + .and_then(|index| child.schema().fields.get(index)) .is_some_and(|field| field.nullable) } _ => false, @@ -1045,6 +1039,6 @@ fn quantile_may_be_empty(state: &OperatorNode) -> bool { global || nullable } // Selecting whole states empties none of them. - _ => selected_from(state).is_some_and(|states| quantile_may_be_empty(states)), + _ => selected_from(state).is_some_and(|states| quantile_may_be_empty(&states)), } } diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs new file mode 100644 index 000000000..b5d0ac29d --- /dev/null +++ b/crates/types/src/ir/operator/input.rs @@ -0,0 +1,118 @@ +//! How the derivation rules read an operator's inputs: one trait, so the +//! same rules (output schema and kind, input validation, sharing, summary +//! coverage) derive a front-end node's schema and a memo group's +//! `GroupProperties` (concepts doc `memo-groups-properties-requirements.md`, +//! section 3). + +use std::rc::Rc; + +use super::node::{Operator, OperatorNode, OperatorResultKind}; +use super::non_asap::NonASAPOp; +use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::schema::Schema; +use crate::ir::SchemaDerivationError; + +/// An input of an operator, as the rules read it. +pub trait OperatorInput: Sized + Clone { + fn schema(&self) -> &Schema; + fn result_kind(&self) -> OperatorResultKind; + /// The input's own operator, its children as inputs again. + fn operator(&self) -> Operator; + /// What the input covers when it is an aggregate group in the memo; + /// `None` for any other input. + fn aggregate_coverage(&self) -> Option<&SummaryCoverage>; +} + +/// A front-end node tree. Node trees carry no summary coverage. +impl OperatorInput for Rc { + fn schema(&self) -> &Schema { + &self.schema + } + + fn result_kind(&self) -> OperatorResultKind { + self.result_kind + } + + fn operator(&self) -> Operator { + self.operator.clone() + } + + fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { + None + } +} + +/// An input of a rule: a real input `I`, or a build the rule constructs +/// only while it derives. +#[derive(Clone)] +pub(crate) enum InputOrTemporaryBuild { + Input(I), + TemporaryBuild(Rc>), +} + +/// A build a rule constructs only while it derives (the whole-window build +/// of a chain, the build a rollup stands for, the build a selection of +/// states stands for): an operator over `I`s with its derived schema and +/// kind. Never stored in the memo. A rule never builds one over another +/// rule's temporary build, so the type does not nest. +pub(crate) struct TemporaryBuild { + operator: Operator>, + schema: Schema, + result_kind: OperatorResultKind, +} + +impl InputOrTemporaryBuild { + /// A non-ASAP `operator` with its derived schema and kind. Only non-ASAP + /// rules derive it, which build no input themselves. + pub(crate) fn build( + operator: NonASAPOp>, + ) -> Result { + let schema = operator.output_schema()?; + Ok(Self::with_schema(Operator::NonASAP(operator), schema)) + } + + /// `operator` with `schema` and its derived kind. + pub(crate) fn with_schema( + operator: Operator>, + schema: Schema, + ) -> Self { + let result_kind = operator.output_kind(); + Self::TemporaryBuild(Rc::new(TemporaryBuild { + operator, + schema, + result_kind, + })) + } +} + +impl OperatorInput for InputOrTemporaryBuild { + fn schema(&self) -> &Schema { + match self { + Self::Input(input) => input.schema(), + Self::TemporaryBuild(build) => &build.schema, + } + } + + fn result_kind(&self) -> OperatorResultKind { + match self { + Self::Input(input) => input.result_kind(), + Self::TemporaryBuild(build) => build.result_kind, + } + } + + fn operator(&self) -> Operator { + match self { + Self::Input(input) => input + .operator() + .map_children(|child| Self::Input(child.clone())), + Self::TemporaryBuild(build) => build.operator.clone(), + } + } + + fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { + match self { + Self::Input(input) => input.aggregate_coverage(), + Self::TemporaryBuild(_) => None, + } + } +} diff --git a/crates/types/src/ir/operator/maintained_population.rs b/crates/types/src/ir/operator/maintained_population.rs index 570bebfa6..f11ab8682 100644 --- a/crates/types/src/ir/operator/maintained_population.rs +++ b/crates/types/src/ir/operator/maintained_population.rs @@ -2,6 +2,8 @@ //! Resource limits, ingestion placement and data structures belong to the executor. use serde::{Deserialize, Serialize}; +use crate::ir::operator::input::OperatorInput; + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CurrentSeriesInput { pub metric: String, @@ -56,35 +58,35 @@ pub struct MaintainedPopulation { impl CurrentSeriesInput { /// Verify the named contract against the canonical maintenance input. - pub(crate) fn matches_node(&self, input: &crate::ir::OperatorNode) -> bool { + pub(crate) fn matches_node(&self, input: &I) -> bool { use crate::ir::operator::Source; use crate::ir::scalar::{CompareOpKind, ScalarValue}; use crate::ir::schema::DataType; use crate::ir::{NonASAPOp, Operator, ScalarExpr, TimeRangeKind}; - let input = match &input.operator { + let input = match input.operator() { Operator::NonASAP(NonASAPOp::TimeRange { range, kind: TimeRangeKind::Instant, child, }) if self.lookback_ms > 0 - && *range == std::time::Duration::from_millis(self.lookback_ms) => + && range == std::time::Duration::from_millis(self.lookback_ms) => { - child.as_ref() + child } Operator::NonASAP(NonASAPOp::TimeRange { .. }) => return false, - _ if self.lookback_ms == 300_000 => input, + _ if self.lookback_ms == 300_000 => input.clone(), _ => return false, }; let Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { metric }, predicates, schema, - }) = &input.operator + }) = input.operator() else { return false; }; if self.metric.is_empty() - || *metric != self.metric + || metric != self.metric || (schema.closed && !schema.has_promql_series_identity()) || schema.time_index.is_none() { @@ -99,7 +101,7 @@ impl CurrentSeriesInput { return false; } let mut matchers = Vec::new(); - for predicate in predicates { + for predicate in &predicates { let ScalarExpr::Compare { left, op, right, .. } = &predicate.0 @@ -138,7 +140,7 @@ impl CurrentSeriesInput { impl MaintainedPopulation { /// Whether `input` is the maintenance input this population declares. - pub(crate) fn matches_node(&self, input: &crate::ir::OperatorNode) -> bool { + pub(crate) fn matches_node(&self, input: &I) -> bool { use crate::ir::operator::Source; use crate::ir::schema::DataType; use crate::ir::{NonASAPOp, Operator}; @@ -149,18 +151,21 @@ impl MaintainedPopulation { value_column, grouping, } => { + let operator = input.operator(); let Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { .. }, schema, .. - }) = &input.operator + }) = &operator else { return false; }; // The same computation, whatever accuracy or timing has - // been attached to the node since. - let same_source = - expected.operator == input.operator && expected.schema == input.schema; + // been attached to the node since. A `Scan` has no input, + // so its operator without inputs is all of it. + let same_source = expected.operator.map_children(|_| ()) + == operator.map_children(|_| ()) + && expected.schema == *input.schema(); same_source && schema.closed && schema diff --git a/crates/types/src/ir/operator/mod.rs b/crates/types/src/ir/operator/mod.rs index f7baca7a9..b9b53b2a4 100644 --- a/crates/types/src/ir/operator/mod.rs +++ b/crates/types/src/ir/operator/mod.rs @@ -4,18 +4,21 @@ //! - [`node`] — [`OperatorNode`](node::OperatorNode): an operator plus the //! planning properties every traversal needs. //! - [`non_asap`] / [`asap`] — ordinary query operators and ASAP operators. -//! - [`operator_properties`] — operator parameter types ([`GroupKeys`], +//! - [`input`] — how the derivation rules read an operator's inputs +//! ([`OperatorInput`](input::OperatorInput)). +//! - [`arguments`] — operator argument types ([`GroupKeys`], //! [`Reduction`], [`Source`], …). //! - [`agg_intent`] — the aggregation-intent vocabulary ([`AggIntent`]). //! - [`maintained_population`] — maintained populations and their evaluations. //! - [`tumbling_window_chain`] — what a tumbling-window chain computes. //! - [`query`] — a query root: an operator result or a standalone scalar. pub mod agg_intent; +pub mod arguments; pub mod asap; +pub mod input; pub mod maintained_population; pub mod node; pub mod non_asap; -pub mod operator_properties; pub mod query; pub mod tumbling_window_chain; @@ -23,7 +26,7 @@ pub use agg_intent::{ agg_has_exact_per_entity_kernel, default_cardinality, default_quantile, AggIntent, MathFunc, TimeFunc, }; -pub use operator_properties::{ +pub use arguments::{ AtModifier, BinaryOpKind, ColState, GroupKeys, GroupSide, InfoMatcher, JoinKind, PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, Source, TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, WindowFrameOffset, diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index d2ae4f643..7551f4d6c 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -1,7 +1,9 @@ //! The unified operator node: one node type before and after ASAP //! optimization. A node is an operator plus the common planning properties -//! every traversal needs (output category and schema, accuracy guarantee, -//! execution timing). +//! every traversal needs (output category and schema, accuracy guarantee). +//! Physical properties (timing, lifetime, schedule) are not on the node: +//! each physical candidate holds its own +//! ([`crate::stages::PhysicalCandidate::properties`]). use std::collections::HashSet; use std::rc::Rc; @@ -9,13 +11,11 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use super::asap::ASAPOp; +use super::input::OperatorInput; use super::non_asap::NonASAPOp; -use crate::ir::properties::execution_data_state::ExecutionTiming; -use crate::ir::properties::guarantee::AccuracyGuarantee; -use crate::ir::properties::summary_coverage::{NodeToSummaryCoverageCache, SummaryCoverage}; +use crate::accuracy::AccuracyGuarantee; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; -use crate::physical::materialization::Materialization; /// The output category of an operator, derived from the operation and its /// inputs. Matching column schemas do not make categories interchangeable. @@ -62,7 +62,9 @@ impl Operator { } } -impl Operator { +/// The derivation rules, one set for every kind of input +/// ([`OperatorInput`]): front-end node trees and memo groups. +impl Operator { pub(crate) fn output_schema(&self) -> Result { match self { Operator::NonASAP(op) => op.output_schema(), @@ -83,6 +85,16 @@ impl Operator { Operator::ASAP(op) => op.validate_inputs(), } } + + /// Whether two equal copies of this operator over the same inputs return + /// the same rows, so they may be one group: an ASAP operator, or a + /// non-ASAP one [`NonASAPOp::is_shareable_when_equal`]. + pub(crate) fn is_shareable_when_equal(&self) -> bool { + match self { + Operator::NonASAP(op) => op.is_shareable_when_equal(), + Operator::ASAP(_) => true, + } + } } /// A node of the logical DAG. Nodes are immutable and shared through `Rc`; @@ -91,19 +103,13 @@ impl Operator { /// /// `schema` and `result_kind` are derived from `operator` and its children /// at construction and retained. `guarantee` is `None` until accuracy -/// assessment establishes one (`None` never means exact). `timing` is `None` -/// until a materialization assignment is applied; export rejects an executable -/// node without one. `materialization` is the option that assignment chose -/// for this node (`NotMaterialized` until then, and for every node whose -/// output is not stored). +/// assessment establishes one (`None` never means exact). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OperatorNode { pub operator: Operator, pub result_kind: OperatorResultKind, pub schema: Schema, pub guarantee: Option, - pub timing: Option, - pub materialization: Materialization, } impl OperatorNode { @@ -112,9 +118,7 @@ impl OperatorNode { /// ASAP operator, ...). pub fn new(operator: Operator) -> Result { let schema = operator.output_schema()?; - let node = Self::with_schema(operator, schema); - node.check_merge(&mut NodeToSummaryCoverageCache::new())?; - Ok(node) + Ok(Self::with_schema(operator, schema)) } /// Build a node with caller-supplied output names and qualifiers. For @@ -127,8 +131,6 @@ impl OperatorNode { result_kind, schema, guarantee: None, - timing: None, - materialization: Materialization::NotMaterialized, } } @@ -144,55 +146,10 @@ impl OperatorNode { self } - pub fn with_timing(mut self, timing: Option) -> Self { - self.timing = timing; - self - } - - pub fn with_materialization(mut self, materialization: Materialization) -> Self { - self.materialization = materialization; - self - } - - /// What this summary state covers; `None` for a node that is not a - /// `SummaryAgg` or a valid `SummaryMerge`. Derived on each call, since - /// the public fields can be edited in place. - #[cfg(any(test, feature = "test-support"))] - pub fn coverage(&self) -> Option { - SummaryCoverage::derive(self).ok() - } - - /// The build whose state this summary-state node carries: a - /// `SummaryAgg` or `SummaryTumblingWindowChain` itself, or the first input's build under - /// `SummaryMerge`s (merged inputs have one definition, which - /// `coverage` checks). `None` under a regrouping - /// merge, whose states no build in the DAG carries (its coverage's - /// `definition` is the build it stands for), and for any other node. - pub fn summary_build(&self) -> Option<&ASAPOp> { - match self.asap()? { - op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) => { - Some(op) - } - ASAPOp::SummaryMerge { - children, - group_by: None, - .. - } => children.first()?.summary_build(), - _ => None, - } - } - - /// A `SummaryMerge` is valid only over inputs with the same definition - /// and disjoint selections. `cache` holds the coverage already derived - /// for nodes of the DAG being validated. - fn check_merge( - &self, - cache: &mut NodeToSummaryCoverageCache, - ) -> Result<(), SchemaDerivationError> { - if matches!(self.asap(), Some(ASAPOp::SummaryMerge { .. })) { - SummaryCoverage::derive_cached(self, cache)?; - } - Ok(()) + /// The build whose state this summary-state node carries + /// ([`summary_build`](super::asap::summary_build)). + pub fn summary_build(self: &Rc) -> Option { + super::asap::summary_build(self) } pub fn non_asap(&self) -> Option<&NonASAPOp> { @@ -237,8 +194,7 @@ impl OperatorNode { /// the old derived schema are retained, for either operator category. A /// change in output arity with such overrides needs an explicit new /// naming assignment. - /// `guarantee`, `timing` and `materialization` depend on the inputs and - /// are cleared. + /// `guarantee` depends on the inputs and is cleared. pub fn with_new_children( &self, f: impl FnMut(&Rc) -> Rc, @@ -302,39 +258,12 @@ impl OperatorNode { out } - /// Validate assigned phases without imposing any particular runtime implementation. - #[cfg(any(test, feature = "test-support"))] - pub fn validate_execution_timing(self: &Rc) -> Result<(), SchemaDerivationError> { - self.validate_structure()?; - for node in Self::reachable(self) { - let timing = node.timing.ok_or_else(|| { - SchemaDerivationError::InvalidScalarSignature( - "execution timing is unassigned".into(), - ) - })?; - if timing == crate::ir::properties::ExecutionTiming::IngestionTime - && node.children().iter().any(|child| { - child.timing != Some(crate::ir::properties::ExecutionTiming::IngestionTime) - }) - { - return Err(SchemaDerivationError::InvalidScalarSignature( - "ingestion-time operation depends on a query-time or unassigned input".into(), - )); - } - } - Ok(()) - } - /// Validate the whole DAG reachable from this node: every operator's /// input contract, scalar typing against the owning operator's input /// schema, and agreement between each retained schema and the one /// derived from the operator. Both operator categories may override field /// names and qualifiers; all structural metadata must match derivation. - /// `timing` may be `None`. pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError> { - // Each summary state's coverage is derived once, not once per merge - // above it; `reachable` keeps every node alive while it is used. - let mut coverage = NodeToSummaryCoverageCache::new(); for node in Self::reachable(self) { if node.schema.time_index.is_some_and(|i| { node.schema.fields.get(i).is_none_or(|f| { @@ -352,7 +281,6 @@ impl OperatorNode { )); } node.operator.validate_inputs()?; - node.check_merge(&mut coverage)?; if node.result_kind != node.operator.output_kind() { return Err(SchemaDerivationError::InvalidScalarSignature( "retained result kind disagrees with operation".into(), diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 7eed22fda..7eb6bbfba 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -6,9 +6,10 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; -use super::node::{OperatorNode, OperatorResultKind}; +use super::input::OperatorInput; +use super::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::agg_intent::AggIntent; -use crate::ir::operator::operator_properties::{ +use crate::ir::operator::arguments::{ BinaryOpKind, GroupKeys, InfoMatcher, JoinKind, Reduction, RelationalSetOpKind, SampleKind, Source, TimeShift, VectorMatch, WindowFrame, WindowFrameUnits, WindowFuncKind, }; @@ -491,7 +492,7 @@ impl NonASAPOp { } } -impl NonASAPOp { +impl NonASAPOp { /// Whether two structurally equal copies of this node may be one shared /// node: every execution over the same input rows returns the same /// output rows, so one copy over their one input can be computed (or @@ -514,7 +515,7 @@ impl NonASAPOp { /// function the IR accepts is deterministic (no `random()`; unregistered /// SQL functions are rejected) and that a deployment's /// `AggIntent::Extension` returns the same rows for the same input. - pub fn is_shareable_when_equal(&self) -> bool { + pub(crate) fn is_shareable_when_equal(&self) -> bool { use NonASAPOp::*; match self { Limit { @@ -524,14 +525,14 @@ impl NonASAPOp { child, } => { (n.is_none() && *offset == 0) - || matches!(child.non_asap(), Some(Sort { partition_by: sorted_by, .. }) - if sorted_by == partition_by) + || matches!(child.operator(), Operator::NonASAP(Sort { partition_by: sorted_by, .. }) + if sorted_by == *partition_by) } // Empty `cols` means every column. Dedup { cols, child } => { cols.is_empty() || cols.iter().collect::>().len() - == child.schema.fields.len() + == child.schema().fields.len() } SQLWindowFunc { func, @@ -610,7 +611,7 @@ impl NonASAPOp { .output_schema(); } let mut output = - aggregate_output_schema(&child.schema, reduction, measures, output_names)?; + aggregate_output_schema(child.schema(), reduction, measures, output_names)?; let offset = reduction.group_keys().map_or(0, |keys| keys.len()); // A filtered quantile is NULL for a group without a matching // row, as its summary estimate is. @@ -625,14 +626,14 @@ impl NonASAPOp { } // Over a relation, a quantile is also NULL for a global // aggregate of no row and a group whose values are all NULL. - if child.result_kind == OperatorResultKind::Relation { + if child.result_kind() == OperatorResultKind::Relation { for (index, measure) in measures.iter().enumerate() { if matches!(measure, AggIntent::Quantile { .. }) && (offset == 0 || measure .input_cols() .iter() - .any(|i| child.schema.fields[*i].nullable)) + .any(|i| child.schema().fields[*i].nullable)) { if let Some(field) = output.fields.get_mut(offset + index) { field.nullable = true; @@ -640,7 +641,7 @@ impl NonASAPOp { } } } - if child.result_kind == OperatorResultKind::Relation { + if child.result_kind() == OperatorResultKind::Relation { for (index, measure) in measures.iter().enumerate() { if matches!( measure, @@ -654,7 +655,7 @@ impl NonASAPOp { || measure .input_cols() .iter() - .any(|i| child.schema.fields[*i].nullable); + .any(|i| child.schema().fields[*i].nullable); if let Some(field) = output.fields.get_mut(offset + index) { field.nullable = nullable; } @@ -671,13 +672,13 @@ impl NonASAPOp { | PromqlSeriesSample { child, .. } | PromqlInfoEnrich { child, .. } | TimeRange { child, .. } - | TimeShift { child, .. } => child.schema.clone(), + | TimeShift { child, .. } => child.schema().clone(), // ρ — relabel preserves every input column and writes one label // `dst` (Utf8): overwritten in place if it already exists, else // appended (nullable). Row-uniqueness is no longer provable. PromqlRelabel { dst, child, .. } => { - let mut out = child.schema.clone(); + let mut out = child.schema().clone(); if let Some(existing) = out.fields.iter_mut().find(|c| c.name == *dst) { existing.dtype = FieldDataType::Plain(DataType::Utf8); existing.nullable = true; @@ -698,7 +699,7 @@ impl NonASAPOp { qualifier, child, } => { - let in_schema = &child.schema; + let in_schema = child.schema(); let fields: Vec = cols .iter() .enumerate() @@ -747,7 +748,7 @@ impl NonASAPOp { } Dedup { cols, child } => { - let mut out = child.schema.clone(); + let mut out = child.schema().clone(); if !cols.is_empty() { out.add_unique_key(cols.clone()); } @@ -758,12 +759,12 @@ impl NonASAPOp { let mut s = children .first() .ok_or(SchemaDerivationError::EmptyConcat)? - .schema + .schema() .clone(); // A column is nullable if any child's is: a rolled-up // grouping key is NULL only in the coarser levels. for child in &children[1..] { - for (field, other) in s.fields.iter_mut().zip(&child.schema.fields) { + for (field, other) in s.fields.iter_mut().zip(&child.schema().fields) { field.nullable |= other.nullable; } } @@ -771,7 +772,7 @@ impl NonASAPOp { s } SetOp { left, .. } => { - let mut s = left.schema.clone(); + let mut s = left.schema().clone(); s.unique_keys.clear(); s } @@ -781,8 +782,8 @@ impl NonASAPOp { left, right, } => { - let l = &left.schema; - let r = &right.schema; + let l = left.schema(); + let r = right.schema(); if matches!(kind, JoinKind::Semi | JoinKind::Anti) { return Ok(Schema { unique_keys: Vec::new(), @@ -825,7 +826,7 @@ impl NonASAPOp { child, .. } => { - let mut out = child.schema.clone(); + let mut out = child.schema().clone(); let arg = args.first().map(|a| a.scalar_type(&out)).transpose()?; let arg_dtype = || { arg.as_ref() @@ -871,7 +872,7 @@ impl NonASAPOp { BinaryOp { lhs, rhs, operator, .. } => { - let mut output = lhs.schema.clone(); + let mut output = lhs.schema().clone(); let grouping = operator .vector_match .as_ref() @@ -883,7 +884,7 @@ impl NonASAPOp { let mut additions = Vec::new(); if right_rows { additions.extend( - rhs.schema + rhs.schema() .fields .iter() .filter(|c| c.plain_dtype() == Some(&DataType::Utf8)) @@ -917,7 +918,7 @@ impl NonASAPOp { Source::Table { .. } => OperatorResultKind::Relation, }, Values { .. } | SQLWindowFunc { .. } => OperatorResultKind::Relation, - TimeRange { child, .. } if child.result_kind == OperatorResultKind::Relation => { + TimeRange { child, .. } if child.result_kind() == OperatorResultKind::Relation => { OperatorResultKind::Relation } TimeRange { kind, .. } => match kind { @@ -931,12 +932,12 @@ impl NonASAPOp { // category (a SQL GROUP BY stays a relation). Aggregate { reduction, child, .. - } => match (reduction, child.result_kind) { + } => match (reduction, child.result_kind()) { (Reduction::PerEntity, OperatorResultKind::RangeVector) | (_, OperatorResultKind::State) => OperatorResultKind::InstantVector, (_, kind) => kind, }, - Project { child, cols, .. } if cols.iter().any(|item| matches!(item.expr, ScalarExpr::Column(i) if child.schema.fields.get(i).is_some_and(|f| !f.is_plain()))) => OperatorResultKind::State, + Project { child, cols, .. } if cols.iter().any(|item| matches!(item.expr, ScalarExpr::Column(i) if child.schema().fields.get(i).is_some_and(|f| !f.is_plain()))) => OperatorResultKind::State, // Selecting whole states by their plain (key) columns keeps them states. Filter { child, pred } if selects_states(child, pred) => OperatorResultKind::State, Filter { child, .. } @@ -947,12 +948,12 @@ impl NonASAPOp { | TimeShift { child, .. } | PromqlRelabel { child, .. } | PromqlInfoEnrich { child, .. } - | PromqlSeriesSample { child, .. } => readable(child.result_kind), - Join { left, .. } | SetOp { left, .. } => readable(left.result_kind), + | PromqlSeriesSample { child, .. } => readable(child.result_kind()), + Join { left, .. } | SetOp { left, .. } => readable(left.result_kind()), Concat { children, .. } => children .first() - .map_or(OperatorResultKind::Relation, |c| readable(c.result_kind)), - BinaryOp { lhs, .. } => readable(lhs.result_kind), + .map_or(OperatorResultKind::Relation, |c| readable(c.result_kind())), + BinaryOp { lhs, .. } => readable(lhs.result_kind()), } } @@ -960,8 +961,8 @@ impl NonASAPOp { /// and its children's output categories. pub(crate) fn validate_inputs(&self) -> Result<(), SchemaDerivationError> { use NonASAPOp::*; - let no_state = |node: &OperatorNode, what: &str| { - if node.result_kind == OperatorResultKind::State { + let no_state = |node: &I, what: &str| { + if node.result_kind() == OperatorResultKind::State { Err(SchemaDerivationError::InvalidScalarSignature(format!( "{what} consumes summary state; read it out first" ))) @@ -970,18 +971,19 @@ impl NonASAPOp { } }; let invalid = |message: &str| SchemaDerivationError::InvalidScalarSignature(message.into()); - let predicate = |pred: &Predicate, scope: &Schema| -> Result<(), SchemaDerivationError> { - if matches!( - pred.0.scalar_type(scope)?.0, - DataType::Bool | DataType::Null - ) { - Ok(()) - } else { - Err(invalid("predicate must be boolean")) - } - }; - let instant = |child: &OperatorNode| -> Result<(), SchemaDerivationError> { - if child.result_kind == OperatorResultKind::InstantVector { + let predicate = + |pred: &Predicate, scope: &Schema| -> Result<(), SchemaDerivationError> { + if matches!( + pred.0.scalar_type(scope)?.0, + DataType::Bool | DataType::Null + ) { + Ok(()) + } else { + Err(invalid("predicate must be boolean")) + } + }; + let instant = |child: &I| -> Result<(), SchemaDerivationError> { + if child.result_kind() == OperatorResultKind::InstantVector { Ok(()) } else { Err(invalid("operation requires an instant vector")) @@ -1025,14 +1027,14 @@ impl NonASAPOp { if !selects_states(child, pred) { no_state(child, "Filter")?; } - predicate(pred, &child.schema)?; + predicate(pred, child.schema())?; } Project { child, cols, .. } => { for col in cols { if let ScalarExpr::Column(index) = col.expr { - columns(&[index], &child.schema)?; + columns(&[index], child.schema())?; } else { - col.expr.scalar_type(&child.schema)?; + col.expr.scalar_type(child.schema())?; } } } @@ -1044,8 +1046,8 @@ impl NonASAPOp { } => { no_state(lhs, "BinaryOp")?; no_state(rhs, "BinaryOp")?; - if lhs.result_kind != rhs.result_kind - || lhs.result_kind == OperatorResultKind::RangeVector + if lhs.result_kind() != rhs.result_kind() + || lhs.result_kind() == OperatorResultKind::RangeVector { return Err(invalid("binary operands have incompatible result kinds")); } @@ -1058,16 +1060,16 @@ impl NonASAPOp { } => { no_state(left, "Join")?; no_state(right, "Join")?; - if left.result_kind != OperatorResultKind::Relation - || right.result_kind != OperatorResultKind::Relation + if left.result_kind() != OperatorResultKind::Relation + || right.result_kind() != OperatorResultKind::Relation { return Err(invalid("SQL join requires relations")); } let scope = Schema::new( - left.schema + left.schema() .fields .iter() - .chain(&right.schema.fields) + .chain(&right.schema().fields) .cloned() .collect(), ); @@ -1076,8 +1078,8 @@ impl NonASAPOp { SetOp { left, right, .. } => { no_state(left, "SetOp")?; no_state(right, "SetOp")?; - if left.result_kind != OperatorResultKind::Relation - || right.result_kind != OperatorResultKind::Relation + if left.result_kind() != OperatorResultKind::Relation + || right.result_kind() != OperatorResultKind::Relation { return Err(invalid("SQL set operation requires relations")); } @@ -1097,16 +1099,16 @@ impl NonASAPOp { } => { no_state(child, "Aggregate")?; if let Reduction::Reduce(keys) = reduction { - columns(keys.keys(), &child.schema)?; + columns(keys.keys(), child.schema())?; } for measure in measures { - columns(&measure.input_cols(), &child.schema)?; + columns(&measure.input_cols(), child.schema())?; } if !filters.is_empty() && filters.len() != measures.len() { return Err(invalid("aggregate filter count differs from measure count")); } for pred in filters.iter().flatten() { - predicate(pred, &child.schema)?; + predicate(pred, child.schema())?; } if let Some(pred) = having { predicate(pred, &self.output_schema()?)?; @@ -1114,23 +1116,23 @@ impl NonASAPOp { } Dedup { child, cols } => { no_state(child, "Dedup")?; - columns(cols, &child.schema)?; + columns(cols, child.schema())?; } Sort { child, keys, partition_by, } => { - columns(partition_by.keys(), &child.schema)?; + columns(partition_by.keys(), child.schema())?; for key in keys { - key.expr.scalar_type(&child.schema)?; + key.expr.scalar_type(child.schema())?; } } Limit { child, partition_by, .. - } => columns(partition_by.keys(), &child.schema)?, + } => columns(partition_by.keys(), child.schema())?, SQLWindowFunc { child, args, @@ -1138,12 +1140,12 @@ impl NonASAPOp { partition_by, .. } => { - if child.result_kind != OperatorResultKind::Relation { + if child.result_kind() != OperatorResultKind::Relation { return Err(invalid("SQL window requires a relation")); } - columns(partition_by.keys(), &child.schema)?; + columns(partition_by.keys(), child.schema())?; for expr in args.iter().chain(order_by.iter().map(|k| &k.expr)) { - expr.scalar_type(&child.schema)?; + expr.scalar_type(child.schema())?; } } PromqlVectorFromScalar(expr) => { @@ -1156,12 +1158,12 @@ impl NonASAPOp { | PromqlSeriesSample { child, .. } => instant(child)?, PromqlRelabel { child, value, .. } => { instant(child)?; - if value.scalar_type(&child.schema)?.0 != DataType::Utf8 { + if value.scalar_type(child.schema())?.0 != DataType::Utf8 { return Err(invalid("label expression requires a string")); } } TimeRange { child, .. } => { - if child.result_kind != OperatorResultKind::Relation { + if child.result_kind() != OperatorResultKind::Relation { instant(child)?; } } @@ -1169,7 +1171,7 @@ impl NonASAPOp { // A relation is a SQL table's rows, shifted under a trailing // window's tumbling window. if !matches!( - child.result_kind, + child.result_kind(), OperatorResultKind::InstantVector | OperatorResultKind::RangeVector | OperatorResultKind::Relation @@ -1187,13 +1189,13 @@ impl NonASAPOp { /// Each input retains its keys only when the equality predicates constrain /// the other input to at most one matching row. In particular this preserves /// grouped sum/count keys through the SQL-average implementation. -fn inner_join_unique_keys( - predicate: &ScalarExpr, +fn inner_join_unique_keys( + predicate: &ScalarExpr, left: &Schema, right: &Schema, ) -> Vec> { use crate::ir::scalar::CompareOpKind; - fn columns(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { + fn columns(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { if let ScalarExpr::Compare { left, op: CompareOpKind::Eq, @@ -1208,7 +1210,7 @@ fn inner_join_unique_keys( } None } - fn equality(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { + fn equality(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { if let Some(pair) = columns(expr) { return Some(pair); } @@ -1241,7 +1243,7 @@ fn inner_join_unique_keys( } None } - fn collect(expr: &ScalarExpr, pairs: &mut Vec<(ColumnId, ColumnId)>) { + fn collect(expr: &ScalarExpr, pairs: &mut Vec<(ColumnId, ColumnId)>) { if let ScalarExpr::BoolAnd(terms) = expr { for term in terms { collect(term, pairs); @@ -1289,11 +1291,11 @@ fn inner_join_unique_keys( /// Whether a `Filter` of `pred` over `child` selects whole summary states: /// `child` is state and `pred` reads only its plain columns, such as group keys. -fn selects_states(child: &OperatorNode, pred: &Predicate) -> bool { - child.result_kind == OperatorResultKind::State +fn selects_states(child: &I, pred: &Predicate) -> bool { + child.result_kind() == OperatorResultKind::State && pred.0.columns_referenced().iter().all(|&column| { child - .schema + .schema() .fields .get(column) .is_some_and(|field| field.is_plain()) @@ -1309,7 +1311,11 @@ fn readable(kind: OperatorResultKind) -> OperatorResultKind { /// Default output-column name for a projection item with no explicit alias: /// a bare column keeps its (schema) name; anything else gets `col_{i}`. -fn default_proj_name(expr: &ScalarExpr, idx: usize, schema: &Schema) -> String { +fn default_proj_name( + expr: &ScalarExpr, + idx: usize, + schema: &Schema, +) -> String { match expr { ScalarExpr::Column(id) => schema .fields @@ -1328,7 +1334,7 @@ pub fn any_measure_filtered(filters: &[Option]) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::ir::operator::operator_properties::{ + use crate::ir::operator::arguments::{ AtModifier, VectorMatchKind, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, }; use crate::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; @@ -1793,7 +1799,7 @@ mod tests { unique_keys: vec![vec![0, 1]], closed: true, }; - let equal = |left, right| ScalarExpr::Compare { + let equal = |left, right| ScalarExpr::>::Compare { left: Box::new(ScalarExpr::Column(left)), op: CompareOpKind::Eq, right: Box::new(ScalarExpr::Column(right)), diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index 8416fa701..aca6285b0 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -1,453 +1,33 @@ -//! Tumbling-window summaries: what an [`ASAPOp::SummaryTumblingWindowChain`] computes, -//! as the plain `SummaryAgg` nodes it stands for. -//! -//! A chain of `n` tumbling windows of width `w` over `child` is tumbling window `i` (newest first, -//! `0 ≤ i < n`): the chain's `SummaryAgg` over a range `TimeRange(w)` over -//! `child` shifted back by `i·w`. A `SummaryMerge` over the chain is the -//! state of one build over `TimeRange(n·w)` (`whole_window_build`) when -//! the family merges (the family-capability table), and the executor runs -//! the chain as its tumbling windows (`expand_tumbling_window_chains`). A merge of only the -//! newest `m < n` tumbling windows (`SummaryMerge::newest_tumbling_windows`) is the state of one -//! build over `TimeRange(m·w)` ([`merged_window_build`]): one chain kept for -//! the longest window serves every query whose window, on the same tumbling windows, -//! ends at the same time and is shorter. -use std::collections::HashMap; -use std::rc::Rc; -use std::time::Duration; - +//! Tumbling-window chains ([`ASAPOp::SummaryTumblingWindowChain`]): one +//! row per window of width `w`, on a grid anchored at the epoch, a state +//! with a time index. A chain has no range and no window count: a +//! `TimeRange` (and `TimeShift`) over it selects the windows a query reads +//! (#1207 PR 1), and how many windows it retains is $h_v$ (#1207 PR 7). use super::asap::ASAPOp; -use super::node::{Operator, OperatorNode}; -use super::non_asap::{NonASAPOp, TimeRangeKind}; -use super::operator_properties::TimeShift; -use super::query::QueryRoot; -use crate::ir::properties::ExecutionTiming; -use crate::ir::SchemaDerivationError; - -fn invalid(message: &str) -> SchemaDerivationError { - SchemaDerivationError::InvalidScalarSignature(message.into()) -} - -/// The `SummaryTumblingWindowChain` a summary-state node carries merged: the chain itself, -/// or the one chain a `SummaryMerge` that keeps its groups reads, in full -/// or its newest tumbling windows (`merged_tumbling_window_count`). `None` for any other node, -/// a rollup (regrouping merge) included. -pub fn merged_chain(state: &OperatorNode) -> Option<&OperatorNode> { - match state.asap()? { - ASAPOp::SummaryTumblingWindowChain { .. } => Some(state), - ASAPOp::SummaryMerge { +use super::input::OperatorInput; +use super::node::Operator; + +/// The `SummaryTumblingWindowChain` a summary state carries merged: the +/// chain itself, or the one chain a `SummaryMerge` that keeps its groups +/// reads. `None` for any other input, a rollup (regrouping merge) included. +pub fn merged_chain(state: &I) -> Option { + match state.operator() { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => Some(state.clone()), + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by: None, .. - } => match children.as_slice() { + }) => match children.as_slice() { [child] if matches!( - child.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) + child.operator(), + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) ) => { - Some(child) + Some(child.clone()) } _ => None, }, _ => None, } } - -/// How many of its chain's tumbling windows, newest first, a summary-state node with -/// a [`merged_chain`] merges: the merge's `newest_tumbling_windows`, otherwise the -/// chain's `tumbling_window_count`. `None` for any other node. -fn merged_tumbling_window_count(state: &OperatorNode) -> Option { - let Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) = merged_chain(state)?.asap() - else { - return None; - }; - match state.asap()? { - ASAPOp::SummaryMerge { - newest_tumbling_windows: Some(newest), - .. - } => Some(*newest), - _ => Some(*tumbling_window_count), - } -} - -/// The `SummaryAgg` over `TimeRange(tumbling_window_count · tumbling_window_width_ms)` over the -/// chain's input whose state the merged tumbling windows of `chain` equal. -pub(crate) fn whole_window_build( - chain: &OperatorNode, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) = chain.asap() - else { - return Err(invalid("expected a tumbling-window chain")); - }; - newest_tumbling_windows_build(chain, *tumbling_window_count) -} - -/// The `SummaryAgg` whose state a summary-state node with a -/// [`merged_chain`] carries: one build over the window of the tumbling windows it -/// merges (`merged_tumbling_window_count`). -pub fn merged_window_build(state: &OperatorNode) -> Result { - let (Some(chain), Some(tumbling_window_count)) = - (merged_chain(state), merged_tumbling_window_count(state)) - else { - return Err(invalid( - "expected merged tumbling windows of one tumbling-window chain", - )); - }; - newest_tumbling_windows_build(chain, tumbling_window_count) -} - -/// The `SummaryAgg` over `TimeRange(tumbling_window_count · tumbling_window_width_ms)` over the -/// chain's input: the state of the chain's newest `tumbling_window_count` tumbling windows. -fn newest_tumbling_windows_build( - chain: &OperatorNode, - tumbling_window_count: u32, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { - child, - tumbling_window_width_ms, - .. - }) = chain.asap() - else { - return Err(invalid("expected a tumbling-window chain")); - }; - let window = tumbling_window_width_ms - .checked_mul(u64::from(tumbling_window_count)) - .ok_or_else(|| invalid("tumbling-window chain window overflows"))?; - build_over(chain, range(Rc::clone(child), window)?) -} - -/// Tumbling window `index` (0 = newest) of `chain`: its `SummaryAgg` over the range -/// `TimeRange(tumbling_window_width_ms)` ending `index · tumbling_window_width_ms` before the -/// chain's input. -pub(crate) fn tumbling_window( - chain: &OperatorNode, - index: u32, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { - child, - tumbling_window_width_ms, - .. - }) = chain.asap() - else { - return Err(invalid("expected a tumbling-window chain")); - }; - let back = i64::try_from(tumbling_window_width_ms.saturating_mul(u64::from(index))) - .map_err(|_| invalid("tumbling window offset overflows"))?; - build_over( - chain, - range(shifted(child, back)?, *tumbling_window_width_ms)?, - ) -} - -/// `child` evaluated `back_ms` earlier: one `TimeShift`, folded into a -/// `TimeShift` directly below. -fn shifted( - child: &Rc, - back_ms: i64, -) -> Result, SchemaDerivationError> { - if back_ms == 0 { - return Ok(Rc::clone(child)); - } - let (shift, input) = match child.non_asap() { - Some(NonASAPOp::TimeShift { - shift, - child: input, - }) => ( - TimeShift { - offset_ms: shift - .offset_ms - .checked_add(back_ms) - .ok_or_else(|| invalid("tumbling window offset overflows"))?, - at: shift.at, - }, - input, - ), - _ => ( - TimeShift { - offset_ms: back_ms, - at: None, - }, - child, - ), - }; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift, - child: Rc::clone(input), - })) -} - -fn range( - child: Rc, - width_ms: u64, -) -> Result, SchemaDerivationError> { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(width_ms), - kind: TimeRangeKind::Range, - child, - })) -} - -/// The chain's `SummaryAgg` over `input`, with the chain's column names. -fn build_over( - chain: &OperatorNode, - input: Rc, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { - family, - input: update, - reduction, - grouping, - filter, - .. - }) = chain.asap() - else { - return Err(invalid("expected a tumbling-window chain")); - }; - let mut build = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { - child: input, - family: family.clone(), - input: update.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }))?; - for (field, kept) in build.schema.fields.iter_mut().zip(&chain.schema.fields) { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - Ok(build) -} - -/// `root` with every `SummaryTumblingWindowChain` expanded into its tumbling windows: a `SummaryMerge` -/// reads each tumbling window of each chain it read, newest first, as plain -/// `SummaryAgg` nodes the executor runs; a merge of the newest `n` tumbling windows -/// reads the first `n`, the same nodes a merge of the whole chain reads. Every other node keeps its -/// guarantee, timing and materialization. Each tumbling window takes its chain's -/// timing and materialization, and the range and shift it reads take the -/// chain's timing: a chain materialized at ingestion time expands into -/// tumbling windows each built and stored at ingestion time. Expanding before timings -/// are applied leaves the tumbling windows untimed. A chain read by anything but a -/// `SummaryMerge` is an error. -#[cfg(any(test, feature = "test-support"))] -pub fn expand_tumbling_window_chains( - root: &Rc, -) -> Result, SchemaDerivationError> { - expand(root, &mut NodeToExpandedNodeCache::default()) -} - -/// A workload with its tumbling-window chains expanded -/// ([`expand_tumbling_window_chains_in_workload`]). -#[derive(Debug, Clone)] -pub struct TumblingWindowChainExpansion { - /// The workload's roots, in order, with every chain expanded. - pub roots: Vec, - /// Each expanded chain with its tumbling windows some merge reads, newest first, - /// in the order the chains were first reached. - pub chains: Vec, - /// Every other node of the input workload the expansion reached, with - /// the node that replaces it (itself when nothing below it changed), - /// so a caller can carry what it knows about a node to the expanded - /// workload. - pub expanded_nodes: Vec<(Rc, Rc)>, -} - -/// One expanded chain: the `SummaryTumblingWindowChain` node of the input workload and the -/// `SummaryAgg` tumbling windows that replace it. -#[derive(Debug, Clone)] -pub struct ExpandedTumblingWindowChain { - /// The chain node, as the input workload holds it. - pub chain: Rc, - /// Its tumbling windows in the expanded workload that some merge reads, newest - /// first. - pub tumbling_windows: Vec>, -} - -/// `expand_tumbling_window_chains` over every root of one workload, with one cache, so -/// a node several roots share stays one node and a chain several merges -/// read expands into one set of tumbling windows. A standalone scalar root has the -/// operators it reads expanded. Also returns each chain's tumbling windows, so a -/// caller can carry what it knows about a chain (e.g. its retention) to -/// them. -pub fn expand_tumbling_window_chains_in_workload( - roots: &[QueryRoot], -) -> Result { - let mut cache = NodeToExpandedNodeCache::default(); - let roots = roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => expand(node, &mut cache).map(QueryRoot::Operator), - QueryRoot::Scalar(expr) => { - let mut error = None; - let expr = expr.map_operator_refs(&mut |node| { - expand(node, &mut cache).unwrap_or_else(|e| { - error.get_or_insert(e); - Rc::clone(node) - }) - }); - match error { - Some(e) => Err(e), - None => Ok(QueryRoot::Scalar(expr)), - } - } - }) - .collect::, _>>()?; - let chains = cache - .chains - .into_iter() - .map(|chain| { - let tumbling_window_count = match chain.asap() { - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => *tumbling_window_count, - _ => unreachable!("only chains are recorded"), - }; - let tumbling_windows = (0..tumbling_window_count) - .map_while(|index| { - cache - .tumbling_window_of - .get(&(Rc::as_ptr(&chain), index)) - .cloned() - }) - .collect(); - ExpandedTumblingWindowChain { - chain, - tumbling_windows, - } - }) - .collect(); - Ok(TumblingWindowChainExpansion { - roots, - chains, - expanded_nodes: cache.expanded, - }) -} - -/// The untimed `node` with `timing` written into it and into every untimed -/// node below it: the nodes [`tumbling_window`] created, over the chain's input, which -/// the timing pass already timed. -fn timed_like(node: &Rc, timing: ExecutionTiming) -> OperatorNode { - OperatorNode::with_schema( - node.operator.map_children(|child| match child.timing { - Some(_) => Rc::clone(child), - None => Rc::new(timed_like(child, timing)), - }), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.clone()) - .with_materialization(node.materialization) - .with_timing(Some(timing)) -} - -/// What one expansion has built: each expanded node by its input's -/// address, and each tumbling window by its chain's address and index, so a chain two -/// merges read expands into one set of tumbling windows. -#[derive(Default)] -struct NodeToExpandedNodeCache { - expanded_node_of: HashMap<*const OperatorNode, Rc>, - tumbling_window_of: HashMap<(*const OperatorNode, u32), Rc>, - /// Each chain expanded, in the order first reached. - chains: Vec>, - /// Each expanded node with its input, in the order expanded. - expanded: Vec<(Rc, Rc)>, -} - -fn expand( - node: &Rc, - cache: &mut NodeToExpandedNodeCache, -) -> Result, SchemaDerivationError> { - if let Some(done) = cache.expanded_node_of.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(done)); - } - let out = match &node.operator { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { - return Err(invalid( - "a tumbling-window chain is read only through a summary merge", - )) - } - Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows, - group_by, - }) => { - let mut inputs = Vec::new(); - for child in children { - match child.asap() { - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => { - for index in 0..newest_tumbling_windows.unwrap_or(*tumbling_window_count) { - let key = (Rc::as_ptr(child), index); - if let Some(done) = cache.tumbling_window_of.get(&key) { - inputs.push(Rc::clone(done)); - continue; - } - let tumbling_window = Rc::new(tumbling_window(child, index)?); - let tumbling_window = match child.timing { - Some(timing) => Rc::new( - timed_like(&tumbling_window, timing) - .with_materialization(child.materialization), - ), - None => tumbling_window, - }; - if index == 0 { - cache.chains.push(Rc::clone(child)); - } - cache - .tumbling_window_of - .insert(key, Rc::clone(&tumbling_window)); - inputs.push(tumbling_window); - } - } - _ => inputs.push(expand(child, cache)?), - } - } - let merge = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: inputs, - newest_tumbling_windows: None, - group_by: group_by.clone(), - }))?; - Rc::new( - OperatorNode::with_schema(merge.operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } - _ => { - let mut changed = false; - let mut children = HashMap::new(); - for child in node.children() { - let new = expand(child, cache)?; - changed |= !Rc::ptr_eq(&new, child); - children.insert(Rc::as_ptr(child), new); - } - if changed { - Rc::new( - OperatorNode::with_schema( - node.operator - .map_children(|child| Rc::clone(&children[&Rc::as_ptr(child)])), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - Rc::clone(node) - } - } - }; - cache - .expanded_node_of - .insert(Rc::as_ptr(node), Rc::clone(&out)); - cache.expanded.push((Rc::clone(node), Rc::clone(&out))); - Ok(out) -} diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs deleted file mode 100644 index ccb07e256..000000000 --- a/crates/types/src/ir/physical_export.rs +++ /dev/null @@ -1,988 +0,0 @@ -//! Physical ASAP DAG (planner-layering stage 2 output). -//! -//! The flattened operators of [`super::flat`], plus the materialization and -//! execution timing (data state) of every node and edge. The input must -//! already be timed ([`crate::ir::properties::timing::apply_materialization_timings`]); export -//! reads each node's timing and materialization and does not re-run -//! data-state validation. The deployment executes the plan as given -//! (#509 Assumption 8): it does not change materialization or timing. - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -use super::flat::{flatten, NodeId}; -use super::scalar::ScalarExpr; -use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; -use crate::ir::operator::non_asap::NonASAPOp; -use crate::ir::operator::operator_properties::Reduction; -use crate::ir::operator::query::QueryRoot; -use crate::ir::properties::execution_data_state::{ - ExecutionDataState, ExecutionDataStateError, ExecutionTiming, -}; -use crate::ir::properties::guarantee::AccuracyGuarantee; -#[cfg(any(test, feature = "test-support"))] -use crate::ir::properties::timing::{ - apply_materialization_timings_to_roots, MaterializationAssignment, -}; -use crate::ir::properties::timing::{data_state, is_materialization_point}; -use crate::ir::schema::{FieldDataType, Schema}; -use crate::physical::materialization::Materialization; - -/// A node's operator, with each child replaced by its node id. -pub type PhysicalASAPOperatorPayload = Operator; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum EdgeRole { - Input, - Left, - Right, - /// The consumer reads the producer from inside one of its scalar - /// expressions (`scalar(v)`, a scalar subquery, `EXISTS`, `IN`). - ScalarRef, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum GroupingEdgeCompatibility { - Identical, - ConsumerCoarsensProducer, - Incompatible, - NotApplicable, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum WindowEdgeCompatibility { - /// Physical lowering must prove equal tumbling-window and query phases or install an - /// exact boundary residual. The logical DAG alone cannot make that claim. - #[serde(rename = "RequiresAlignedPanePhaseOrExactBoundaryResidual")] - RequiresAlignedTumblingWindowPhaseOrExactWindowEdgeResidual, - NotApplicable, -} - -pub type PhysicalASAPNodeId = NodeId; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalASAPDAGNode { - pub id: PhysicalASAPNodeId, - pub payload: PhysicalASAPOperatorPayload, - /// Phase is a placement choice for every operator, independent of payload kind. - pub output_state: ExecutionDataState, - /// Whether and where the deployment stores this node's output: the - /// option the materialization assignment chose for the sub-DAG this node - /// produces. Any node may be stored (#509 §Materialization); a node with - /// no assignment is `NotMaterialized`. - pub materialization: Materialization, - pub output_schema: Schema, - pub guarantee: Option, - /// For a materialized view (a node whose materialization keeps its - /// output across evaluations), how far back from the newest evaluation - /// time its output must stay available once refreshed; for a query-time - /// `Scan`, how far back its source's rows must stay available (`None` - /// when it reads them unbounded). `None` on every other node, and on - /// every node of a DAG exported before installation sets it - /// ([`PhysicalASAPDAG::set_retention`]). A - /// node field rather than part of [`Materialization`], because Stage 2 - /// enumerates materializations before Stage 3 knows retention. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub retention_ms: Option, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalASAPDAGEdge { - pub producer: PhysicalASAPNodeId, - pub consumer: PhysicalASAPNodeId, - pub role: EdgeRole, - pub(crate) intermediate_schema: Schema, - pub(crate) data_state: ExecutionDataState, - pub(crate) grouping: GroupingEdgeCompatibility, - pub(crate) window: WindowEdgeCompatibility, -} - -#[cfg(any(test, feature = "test-support"))] -impl PhysicalASAPDAGEdge { - pub fn new( - producer: PhysicalASAPNodeId, - consumer: PhysicalASAPNodeId, - role: EdgeRole, - intermediate_schema: Schema, - data_state: ExecutionDataState, - grouping: GroupingEdgeCompatibility, - window: WindowEdgeCompatibility, - ) -> Self { - Self { - producer, - consumer, - role, - intermediate_schema, - data_state, - grouping, - window, - } - } - - pub fn intermediate_schema(&self) -> &Schema { - &self.intermediate_schema - } - - /// For tests that change a compiled DAG's edge schema. - pub fn intermediate_schema_mut(&mut self) -> &mut Schema { - &mut self.intermediate_schema - } - - pub fn data_state(&self) -> ExecutionDataState { - self.data_state - } - - /// For tests that retime a compiled DAG's edges. - pub fn data_state_mut(&mut self) -> &mut ExecutionDataState { - &mut self.data_state - } - - pub fn grouping(&self) -> GroupingEdgeCompatibility { - self.grouping - } - - pub fn window(&self) -> WindowEdgeCompatibility { - self.window - } -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalASAPDAG { - pub nodes: Vec, - pub edges: Vec, - /// Semantic workload roots. Physical query/precompute sinks are selected - /// downstream by the control plane. - /// One root per operator query of the batch, in workload order. - pub roots: Vec, - /// One expression per standalone scalar query, keyed by the query's - /// position in the batch, in increasing order. Its operator references - /// name nodes of this DAG. A standalone scalar query is not a node (#511 - /// §2.3), so query `i` of the batch is the scalar root keyed `i` when - /// there is one, and otherwise the next entry of `roots`. - pub scalar_roots: Vec<(usize, ScalarExpr)>, -} - -#[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum PhysicalASAPDAGValidationError { - #[error( - "node {node:?} runs at {timing:?}, which its materialization {materialization:?} does not" - )] - MaterializationTimingMismatch { - node: PhysicalASAPNodeId, - timing: ExecutionTiming, - materialization: Materialization, - }, - #[error("ingestion node {consumer:?} depends on query node {producer:?}")] - QueryDependencyInIngestion { - producer: PhysicalASAPNodeId, - consumer: PhysicalASAPNodeId, - }, - #[error( - "node {0:?} has a retention_ms but is neither a materialized view nor a query-time scan" - )] - UnexpectedRetention(PhysicalASAPNodeId), - #[error("duplicate physical ASAP node id {0:?}")] - DuplicateNodeId(PhysicalASAPNodeId), - #[error("physical ASAP DAG root {0:?} does not name a node")] - MissingRoot(PhysicalASAPNodeId), - #[error("edge endpoint {0:?} does not name a node")] - MissingEdgeEndpoint(PhysicalASAPNodeId), - #[error("edge {producer:?}->{consumer:?} schema differs from producer output")] - EdgeSchemaMismatch { - producer: PhysicalASAPNodeId, - consumer: PhysicalASAPNodeId, - }, - #[error("edge {producer:?}->{consumer:?} data state differs from producer output")] - EdgeDataStateMismatch { - producer: PhysicalASAPNodeId, - consumer: PhysicalASAPNodeId, - }, - #[error("incoming edges of node {consumer:?} do not match its operator's inputs")] - EdgeRoleMismatch { consumer: PhysicalASAPNodeId }, - #[error("physical ASAP DAG has no query roots")] - NoRoots, - #[error("scalar root keyed {0} is out of order or outside the batch")] - ScalarRootPosition(usize), - #[error("physical ASAP DAG contains a cycle")] - Cycle, - #[error("physical ASAP node {0:?} is not reachable from the root")] - UnreachableNode(PhysicalASAPNodeId), - #[error("summary aggregate node {node:?} output schema does not contain its declared family")] - SummaryFamilySchemaMismatch { node: PhysicalASAPNodeId }, - #[error( - "summary aggregate node {node:?} declares grouping inconsistent with its sketch state" - )] - SummaryGroupingMismatch { node: PhysicalASAPNodeId }, -} - -/// Which rows of a `Scan` its readers read, back from an evaluation time `T` -/// ([`PhysicalASAPDAG::scan_window`]). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum ScanWindow { - /// The rows with time in `(T - end_offset_ms - width_ms, T - - /// end_offset_ms]`. - Within { end_offset_ms: i64, width_ms: i64 }, - /// Rows at any time. - Unbounded, -} - -impl ScanWindow { - /// The smallest window holding both. - pub(crate) fn union(self, other: Self) -> Self { - let ( - Self::Within { - end_offset_ms: a_end, - width_ms: a_width, - }, - Self::Within { - end_offset_ms: b_end, - width_ms: b_width, - }, - ) = (self, other) - else { - return Self::Unbounded; - }; - let end_offset_ms = a_end.min(b_end); - let start_back_ms = a_end - .saturating_add(a_width) - .max(b_end.saturating_add(b_width)); - Self::Within { - end_offset_ms, - width_ms: start_back_ms.saturating_sub(end_offset_ms), - } - } - - /// How far back from `T` the oldest row read lies, `end_offset_ms + - /// width_ms` (0 for a window ending after `T` that reaches no further - /// back); `None` when unbounded. - pub fn lookback_ms(self) -> Option { - match self { - Self::Within { - end_offset_ms, - width_ms, - } => Some(u64::try_from(end_offset_ms.saturating_add(width_ms)).unwrap_or(0)), - Self::Unbounded => None, - } - } - - /// `(start, end]` at evaluation time `evaluation_time_ms`; `None` when - /// unbounded. - pub fn bounds_ms(self, evaluation_time_ms: i64) -> Option<(i64, i64)> { - match self { - Self::Within { - end_offset_ms, - width_ms, - } => { - let end = evaluation_time_ms.saturating_sub(end_offset_ms); - Some((end.saturating_sub(width_ms), end)) - } - Self::Unbounded => None, - } - } -} - -/// The rows the `TimeRange` `time_range` keeps, back from the evaluation -/// time `T`: `(T - s - w, T - s]` for a `TimeRange(w)` over `TimeShift`s -/// whose offsets sum to `s`, the window [`PhysicalASAPDAG::scan_window`] -/// counts for it. `Unbounded` when one of those shifts has an `@` anchor, -/// or `time_range` is not a `TimeRange`. `payload_of` gives each node's -/// operator by id; an indexed lookup keeps this O(shifts), so a caller -/// asking for every `TimeRange` of a tumbling-window-expanded DAG stays linear. -pub fn time_range_window<'a>( - time_range: PhysicalASAPNodeId, - payload_of: impl Fn(PhysicalASAPNodeId) -> Option<&'a PhysicalASAPOperatorPayload>, -) -> ScanWindow { - let Some(Operator::NonASAP(NonASAPOp::TimeRange { range, child, .. })) = payload_of(time_range) - else { - return ScanWindow::Unbounded; - }; - let mut end_offset_ms = 0i64; - let mut below = payload_of(*child); - while let Some(Operator::NonASAP(NonASAPOp::TimeShift { shift, child })) = below { - if shift.at.is_some() { - return ScanWindow::Unbounded; - } - end_offset_ms = end_offset_ms.saturating_add(shift.offset_ms); - below = payload_of(*child); - } - ScanWindow::Within { - end_offset_ms, - width_ms: i64::try_from(range.as_millis()).unwrap_or(i64::MAX), - } -} - -/// How long before the evaluation time `node` is evaluated: the sum of the -/// offsets of the `TimeShift`s from `node` down; `None` when one of them has -/// an `@` anchor or the sum overflows. -pub(crate) fn offset_below<'a>( - node: PhysicalASAPNodeId, - payload_of: impl Fn(PhysicalASAPNodeId) -> Option<&'a PhysicalASAPOperatorPayload>, -) -> Option { - let mut end_offset_ms = 0i64; - let mut below = payload_of(node); - while let Some(Operator::NonASAP(NonASAPOp::TimeShift { shift, child })) = below { - if shift.at.is_some() { - return None; - } - end_offset_ms = end_offset_ms.checked_add(shift.offset_ms)?; - below = payload_of(*child); - } - Some(end_offset_ms) -} - -/// Why `PhysicalASAPDAG::with_every_node_at_query_time` failed. -#[derive(Debug, Error)] -pub enum ReferencePlanError { - /// Bad input: the DAG is not a valid physical ASAP DAG. - #[error("invalid physical ASAP DAG: {0}")] - InvalidDag(#[from] PhysicalASAPDAGValidationError), - /// Bad input: the plan cannot run at query time (a data-state rule of - /// the timing pass fails). - #[error("the plan does not run at query time: {0}")] - Timing(#[from] ExecutionDataStateError), -} - -impl PhysicalASAPDAG { - /// This DAG's plan with nothing materialized: the same operators, - /// summaries and summary parameters, with every sub-DAG - /// `NotMaterialized` and so every node at query time (the default - /// [`MaterializationAssignment`]). Node ids are renumbered, and `roots` - /// and `scalar_roots` keep their order. - /// - /// The correctness reference of a selected plan: a deployment runtime - /// compares the answers of the plan it installed with the executor - /// running this one over raw data, which needs the raw data the - /// installed plan may not keep. - #[cfg(any(test, feature = "test-support"))] - pub fn with_every_node_at_query_time(&self) -> Result { - let roots = self.untimed_query_roots()?; - let timed = - apply_materialization_timings_to_roots(&roots, &MaterializationAssignment::default())?; - Ok(compile_physical_asap_workload(&timed)?) - } - - /// Every node of this DAG as IR, by node id, untimed and - /// `NotMaterialized` with its exported guarantee: what a caller derives - /// IR properties of a plan file's node from, such as its - /// [`SummaryCoverage`](crate::ir::properties::summary_coverage::SummaryCoverage). - /// Shared inputs stay shared. Validates the DAG first. - pub(crate) fn operator_nodes( - &self, - ) -> Result>, PhysicalASAPDAGValidationError> { - self.validate()?; - let by_id: HashMap = - self.nodes.iter().map(|node| (node.id, node)).collect(); - let mut rebuilt: HashMap> = HashMap::new(); - // Children first, without recursion: `validate` rejected cycles. - for node in &self.nodes { - let mut pending = vec![(node.id, false)]; - while let Some((id, children_done)) = pending.pop() { - if rebuilt.contains_key(&id) { - continue; - } - let node = by_id[&id]; - if !children_done { - pending.push((id, true)); - pending.extend(node.payload.children().into_iter().map(|c| (*c, false))); - continue; - } - let operator = node - .payload - .map_children(|child| Rc::clone(&rebuilt[child])); - let restored = OperatorNode::with_schema(operator, node.output_schema.clone()) - .with_guarantee(node.guarantee.clone()); - rebuilt.insert(id, Rc::new(restored)); - } - } - Ok(rebuilt) - } - - /// The rows of the `Scan` `scan` its readers read, back from the - /// evaluation time `T`: the union of each reader's window. A reader - /// reaches the `Scan` through `TimeShift`s without an `@` anchor - /// (offsets summing to `s`) and is - /// - a `TimeRange(w)`, range or instant: `(T - s - w, T - s]`; - /// - a `SummaryTumblingWindowChain` of `n` tumbling windows of width `w`: its whole window `(T - s - - /// n·w, T - s]`, or only its newest tumbling window `(T - s - w, T - s]` for a - /// chain in `kept_chains` (kept across evaluations, so each - /// evaluation builds one tumbling window). - /// - /// `Unbounded` when a root, any other reader, or a shift with an `@` - /// anchor reads the `Scan` or one of those shifts, or nothing reads it. - pub fn scan_window( - &self, - scan: PhysicalASAPNodeId, - kept_chains: &HashSet, - ) -> ScanWindow { - self.scan_windows([scan], kept_chains)[&scan] - } - - /// [`Self::scan_window`] of each of `scans`, by id, with the DAG - /// indexed once for all of them: O(nodes + edges + the readers walked). - pub fn scan_windows( - &self, - scans: impl IntoIterator, - kept_chains: &HashSet, - ) -> HashMap { - let payloads = self.payloads_by_id(); - let mut consumers: HashMap> = HashMap::new(); - for edge in &self.edges { - consumers - .entry(edge.producer) - .or_default() - .push(edge.consumer); - } - let roots: HashSet = self.roots.iter().copied().collect(); - let window_of = |scan: PhysicalASAPNodeId| { - let mut window: Option = None; - let mut pending = vec![(scan, 0i64)]; - while let Some((id, end_offset_ms)) = pending.pop() { - if roots.contains(&id) { - return ScanWindow::Unbounded; - } - for &consumer in consumers.get(&id).into_iter().flatten() { - let width_ms = match payloads.get(&consumer) { - Some(Operator::NonASAP(NonASAPOp::TimeShift { shift, .. })) - if shift.at.is_none() => - { - pending.push((consumer, end_offset_ms.saturating_add(shift.offset_ms))); - continue; - } - Some(Operator::NonASAP(NonASAPOp::TimeRange { range, .. })) => { - range.as_millis() - } - Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - })) if kept_chains.contains(&consumer) => { - u128::from(*tumbling_window_width_ms) - } - Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - })) => { - u128::from(*tumbling_window_width_ms) - * u128::from(*tumbling_window_count) - } - _ => return ScanWindow::Unbounded, - }; - let read = ScanWindow::Within { - end_offset_ms, - width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), - }; - window = Some(window.map_or(read, |window| window.union(read))); - } - } - window.unwrap_or(ScanWindow::Unbounded) - }; - scans - .into_iter() - .map(|scan| (scan, window_of(scan))) - .collect() - } - - /// The interval of data the output of `node` summarizes, back from the - /// evaluation time `T`: the union of the windows - /// ([`time_range_window`]) of every `TimeRange` below it, and of every - /// unexpanded `SummaryTumblingWindowChain` (all its tumbling windows). For an expanded tumbling window, its - /// tumbling window; for a view built from tumbling windows, all of them. `Unbounded` when no - /// `TimeRange` or chain is below it or one of them is unbounded (a whole - /// SQL table, an `@` anchor). O(nodes + edges). - #[cfg(any(test, feature = "test-support"))] - pub fn view_window(&self, node: PhysicalASAPNodeId) -> ScanWindow { - self.view_windows([node])[&node] - } - - /// `Self::view_window` of each of `nodes`, by id, with the DAG indexed - /// once for all of them: O(nodes + edges + the nodes walked below each). - /// It walks each operator's inputs (a valid DAG's edges), so it also - /// answers for a DAG of nodes alone. - pub fn view_windows( - &self, - nodes: impl IntoIterator, - ) -> HashMap { - let payloads = self.payloads_by_id(); - let window_of = |node: PhysicalASAPNodeId| { - let mut seen = HashSet::new(); - let mut pending = vec![node]; - let mut window: Option = None; - while let Some(id) = pending.pop() { - if !seen.insert(id) { - continue; - } - let read = match payloads.get(&id) { - Some(Operator::NonASAP(NonASAPOp::TimeRange { .. })) => { - time_range_window(id, |id| payloads.get(&id).copied()) - } - Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child, - tumbling_window_width_ms, - tumbling_window_count, - .. - })) => { - let width_ms = - tumbling_window_width_ms.checked_mul(u64::from(*tumbling_window_count)); - match ( - offset_below(*child, |id| payloads.get(&id).copied()), - width_ms, - ) { - (Some(end_offset_ms), Some(width_ms)) => ScanWindow::Within { - end_offset_ms, - width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), - }, - _ => ScanWindow::Unbounded, - } - } - Some(payload) => { - pending.extend(payload.children().into_iter().copied()); - continue; - } - None => continue, - }; - window = Some(window.map_or(read, |window| window.union(read))); - } - window.unwrap_or(ScanWindow::Unbounded) - }; - nodes - .into_iter() - .map(|node| (node, window_of(node))) - .collect() - } - - /// Each node's operator, by id. - pub(crate) fn payloads_by_id( - &self, - ) -> HashMap { - self.nodes - .iter() - .map(|node| (node.id, &node.payload)) - .collect() - } - - /// The IR this DAG exports, one root per query in batch order, every - /// node untimed and `NotMaterialized`: the inverse of - /// [`compile_physical_asap_workload`] up to `Rc` identity and timing. - #[cfg(any(test, feature = "test-support"))] - fn untimed_query_roots(&self) -> Result, PhysicalASAPDAGValidationError> { - let rebuilt = self.operator_nodes()?; - let mut operator_roots = self.roots.iter(); - let mut scalar_roots = self.scalar_roots.iter().peekable(); - let mut roots = Vec::with_capacity(self.roots.len() + self.scalar_roots.len()); - for position in 0..self.roots.len() + self.scalar_roots.len() { - match scalar_roots.peek() { - Some((at, expr)) if *at == position => { - roots.push(QueryRoot::Scalar( - expr.map_operator_refs(&mut |id| Rc::clone(&rebuilt[id])), - )); - scalar_roots.next(); - } - _ => { - let id = operator_roots.next().expect("validated root positions"); - roots.push(QueryRoot::Operator(Rc::clone(&rebuilt[id]))); - } - } - } - Ok(roots) - } - - pub fn validate(&self) -> Result<(), PhysicalASAPDAGValidationError> { - let mut nodes = HashMap::new(); - for node in &self.nodes { - if nodes.insert(node.id, node).is_some() { - return Err(PhysicalASAPDAGValidationError::DuplicateNodeId(node.id)); - } - // A sub-DAG output runs when its materialization says; any other - // node is not stored and runs when its consumer runs. - if (is_materialization_point(&node.payload) - || node.materialization != Materialization::NotMaterialized) - && node.materialization.timing() != node.output_state.timing - { - return Err( - PhysicalASAPDAGValidationError::MaterializationTimingMismatch { - node: node.id, - timing: node.output_state.timing, - materialization: node.materialization, - }, - ); - } - if node.retention_ms.is_some() - && !node.materialization.is_materialized_view() - && !super::materialized_view::is_query_time_scan(node) - { - return Err(PhysicalASAPDAGValidationError::UnexpectedRetention(node.id)); - } - if let Operator::ASAP(ASAPOp::SummaryAgg { - family, grouping, .. - }) = &node.payload - { - let mut found_family = false; - for field in &node.output_schema.fields { - if &field.dtype == family { - found_family = true; - } - if let FieldDataType::Sketch(_, schema_grouping) = &field.dtype { - if schema_grouping != grouping { - return Err(PhysicalASAPDAGValidationError::SummaryGroupingMismatch { - node: node.id, - }); - } - } - } - if !found_family { - return Err( - PhysicalASAPDAGValidationError::SummaryFamilySchemaMismatch { - node: node.id, - }, - ); - } - } - } - if self.roots.is_empty() && self.scalar_roots.is_empty() { - return Err(PhysicalASAPDAGValidationError::NoRoots); - } - let queries = self.roots.len() + self.scalar_roots.len(); - let mut next = 0; - for (position, _) in &self.scalar_roots { - if *position < next || *position >= queries { - return Err(PhysicalASAPDAGValidationError::ScalarRootPosition( - *position, - )); - } - next = position + 1; - } - // Operator roots and the nodes scalar roots read. - let root_ids: Vec = self - .roots - .iter() - .copied() - .chain( - self.scalar_roots - .iter() - .flat_map(|(_, expr)| expr.operator_refs().into_iter().copied()), - ) - .collect(); - for root in &root_ids { - if !nodes.contains_key(root) { - return Err(PhysicalASAPDAGValidationError::MissingRoot(*root)); - } - } - // Each node's incoming edges are exactly its operator's inputs, with - // the role of each input's position, so a swapped `Left`/`Right` - // cannot reorder the inputs of a binary operator. - // Count input occurrences instead of scanning a vector for each edge. - // Tumbling-window merges can have tens of thousands of distinct inputs; linear - // searches made validation quadratic in that fan-in. Multiplicities - // still matter when the same producer occupies several input slots. - let mut expected: HashMap<_, HashMap<_, usize>> = self - .nodes - .iter() - .map(|node| { - let mut counts = HashMap::new(); - for input in edge_roles(&node.payload) { - *counts.entry(input).or_insert(0) += 1; - } - (node.id, counts) - }) - .collect(); - for edge in &self.edges { - let inputs = expected.get_mut(&edge.consumer).ok_or( - PhysicalASAPDAGValidationError::MissingEdgeEndpoint(edge.consumer), - )?; - let count = inputs - .get_mut(&(edge.producer, edge.role)) - .filter(|count| **count > 0) - .ok_or(PhysicalASAPDAGValidationError::EdgeRoleMismatch { - consumer: edge.consumer, - })?; - *count -= 1; - } - if let Some((consumer, _)) = expected - .iter() - .find(|(_, inputs)| inputs.values().any(|count| *count != 0)) - { - return Err(PhysicalASAPDAGValidationError::EdgeRoleMismatch { - consumer: *consumer, - }); - } - let mut children: HashMap> = HashMap::new(); - for edge in &self.edges { - let producer = nodes.get(&edge.producer).ok_or( - PhysicalASAPDAGValidationError::MissingEdgeEndpoint(edge.producer), - )?; - if !nodes.contains_key(&edge.consumer) { - return Err(PhysicalASAPDAGValidationError::MissingEdgeEndpoint( - edge.consumer, - )); - } - if producer.output_state.timing == ExecutionTiming::QueryTime - && nodes[&edge.consumer].output_state.timing == ExecutionTiming::IngestionTime - { - return Err(PhysicalASAPDAGValidationError::QueryDependencyInIngestion { - producer: edge.producer, - consumer: edge.consumer, - }); - } - if edge.intermediate_schema != producer.output_schema { - return Err(PhysicalASAPDAGValidationError::EdgeSchemaMismatch { - producer: edge.producer, - consumer: edge.consumer, - }); - } - if edge.data_state != producer.output_state { - return Err(PhysicalASAPDAGValidationError::EdgeDataStateMismatch { - producer: edge.producer, - consumer: edge.consumer, - }); - } - children - .entry(edge.consumer) - .or_default() - .push(edge.producer); - } - fn visit( - id: PhysicalASAPNodeId, - children: &HashMap>, - visiting: &mut HashSet, - visited: &mut HashSet, - ) -> bool { - if visited.contains(&id) { - return true; - } - if !visiting.insert(id) { - return false; - } - if children - .get(&id) - .into_iter() - .flatten() - .any(|child| !visit(*child, children, visiting, visited)) - { - return false; - } - visiting.remove(&id); - visited.insert(id); - true - } - let mut visited = HashSet::new(); - for root in &root_ids { - if !visit(*root, &children, &mut HashSet::new(), &mut visited) { - return Err(PhysicalASAPDAGValidationError::Cycle); - } - } - fn mark( - id: PhysicalASAPNodeId, - children: &HashMap>, - reachable: &mut HashSet, - ) { - if !reachable.insert(id) { - return; - } - for child in children.get(&id).into_iter().flatten() { - mark(*child, children, reachable); - } - } - let mut reachable = HashSet::new(); - for root in &root_ids { - mark(*root, &children, &mut reachable); - } - if let Some(id) = nodes.keys().find(|id| !reachable.contains(id)) { - return Err(PhysicalASAPDAGValidationError::UnreachableNode(*id)); - } - Ok(()) - } -} - -// ── Compilation from the IR ────────────────────────────────────────────── - -/// Compiler-local identity assignment. It deliberately retains `Rc` handles -/// and is not serialized; deployed artifacts persist the physical ASAP node ID -/// together with their physical materialization/query IDs. -#[derive(Debug, Clone)] -pub struct PhysicalASAPNodeIdentityMap { - nodes_by_id: Vec>, -} - -impl PhysicalASAPNodeIdentityMap { - #[cfg(any(test, feature = "test-support"))] - pub fn node_id(&self, node: &Rc) -> Option { - self.nodes_by_id - .iter() - .position(|candidate| Rc::ptr_eq(candidate, node)) - } - - pub fn operator_node(&self, id: PhysicalASAPNodeId) -> Option<&Rc> { - self.nodes_by_id.get(id) - } -} - -#[derive(Debug, Clone)] -pub struct PhysicalASAPDAGCompilation { - pub dag: PhysicalASAPDAG, - pub node_ids: PhysicalASAPNodeIdentityMap, -} - -#[cfg(any(test, feature = "test-support"))] -pub fn compile_physical_asap_dag( - root: &Rc, -) -> Result { - Ok(compile_physical_asap_dag_with_node_ids(root)?.dag) -} - -/// Export the timed DAG below `root`. Every reachable node must carry a -/// timing (see [`crate::ir::properties::timing::apply_materialization_timings`]); the data-state -/// rules were checked by that pass and are not re-run here. -#[cfg(any(test, feature = "test-support"))] -pub(crate) fn compile_physical_asap_dag_with_node_ids( - root: &Rc, -) -> Result { - compile_physical_asap_workload_with_node_ids(&[QueryRoot::Operator(Rc::clone(root))]) -} - -/// Export a timed batch as one DAG: operator queries become `roots` and -/// standalone scalar queries `scalar_roots`, keyed by their position in the -/// batch. -pub fn compile_physical_asap_workload( - roots: &[QueryRoot], -) -> Result { - Ok(compile_physical_asap_workload_with_node_ids(roots)?.dag) -} - -pub fn compile_physical_asap_workload_with_node_ids( - roots: &[QueryRoot], -) -> Result { - let (flat, nodes_by_id) = flatten(roots); - let mut scalar_roots = Vec::new(); - let roots = flat - .roots - .into_iter() - .enumerate() - .filter_map(|(position, root)| match root { - QueryRoot::Operator(id) => Some(id), - QueryRoot::Scalar(expr) => { - scalar_roots.push((position, expr)); - None - } - }) - .collect(); - let mut nodes: Vec = Vec::with_capacity(flat.nodes.len()); - let mut edges = Vec::new(); - for (id, flat_node) in flat.nodes.into_iter().enumerate() { - let node = &nodes_by_id[id]; - let output_state = data_state(node).ok_or(ExecutionDataStateError::UntimedNode { - operator: node.operator.kind_name(), - })?; - // Children come before their parents, so every producer is already in `nodes`. - for (producer, role) in edge_roles(&flat_node.operator) { - let producer_state = nodes[producer].output_state; - let ingestion_dependency = producer_state.timing == ExecutionTiming::IngestionTime - && output_state.timing == ExecutionTiming::IngestionTime; - edges.push(PhysicalASAPDAGEdge { - producer, - consumer: id, - role, - intermediate_schema: nodes[producer].output_schema.clone(), - data_state: producer_state, - grouping: grouping_compatibility(&nodes[producer].payload, &flat_node.operator), - window: if ingestion_dependency { - WindowEdgeCompatibility::RequiresAlignedTumblingWindowPhaseOrExactWindowEdgeResidual - } else { - WindowEdgeCompatibility::NotApplicable - }, - }); - } - nodes.push(PhysicalASAPDAGNode { - id, - payload: flat_node.operator, - output_state, - materialization: flat_node.materialization, - output_schema: flat_node.schema, - guarantee: flat_node.guarantee, - retention_ms: None, - }); - } - let dag = PhysicalASAPDAG { - nodes, - edges, - roots, - scalar_roots, - }; - dag.validate().map_err(ExecutionDataStateError::from)?; - Ok(PhysicalASAPDAGCompilation { - dag, - node_ids: PhysicalASAPNodeIdentityMap { nodes_by_id }, - }) -} - -/// Each input of `operator` with its edge role: the operator inputs, then the -/// nodes read by its scalar expressions (the order of [`Operator::children`]). -/// A node referenced several times from the scalar expressions is one -/// `ScalarRef` input, as cost lowering reads it once; ids are per node -/// pointer, so this dedupes by pointer. -fn edge_roles(operator: &Operator) -> Vec<(C, EdgeRole)> { - use EdgeRole::*; - let inputs: Vec = match operator { - Operator::NonASAP(op) => match op { - NonASAPOp::Join { .. } | NonASAPOp::SetOp { .. } | NonASAPOp::BinaryOp { .. } => { - vec![Left, Right] - } - NonASAPOp::Concat { children, .. } => vec![Input; children.len()], - NonASAPOp::Scan { .. } - | NonASAPOp::Values { .. } - | NonASAPOp::PromqlVectorFromScalar(_) => vec![], - _ => vec![Input], - }, - Operator::ASAP(op) => match op { - ASAPOp::SummarySubtract { .. } | ASAPOp::SummaryJoin { .. } => vec![Left, Right], - ASAPOp::SummaryMerge { children, .. } => vec![Input; children.len()], - _ => vec![Input], - }, - }; - let children = operator.children(); - let (direct, scalar_refs) = children.split_at(inputs.len()); - let mut edges: Vec<(C, EdgeRole)> = direct.iter().map(|c| **c).zip(inputs).collect(); - for producer in scalar_refs { - if !edges.contains(&(**producer, ScalarRef)) { - edges.push((**producer, ScalarRef)); - } - } - edges -} - -fn grouping_compatibility( - producer: &Operator, - consumer: &Operator, -) -> GroupingEdgeCompatibility { - let ( - Operator::ASAP(ASAPOp::SummaryAgg { - reduction: producer, - .. - }), - Operator::ASAP(ASAPOp::SummaryAgg { - reduction: consumer, - .. - }), - ) = (producer, consumer) - else { - return GroupingEdgeCompatibility::NotApplicable; - }; - match (producer, consumer) { - (p, c) if p == c => GroupingEdgeCompatibility::Identical, - (Reduction::PerEntity, Reduction::Reduce(_)) => { - GroupingEdgeCompatibility::ConsumerCoarsensProducer - } - (Reduction::Reduce(p), Reduction::Reduce(c)) - if !p.is_without() && !c.is_without() && c.iter().all(|key| p.contains(key)) => - { - GroupingEdgeCompatibility::ConsumerCoarsensProducer - } - _ => GroupingEdgeCompatibility::Incompatible, - } -} diff --git a/crates/types/src/ir/properties/execution_data_state.rs b/crates/types/src/ir/properties/execution_data_state.rs index e3a2cda8e..287e5035c 100644 --- a/crates/types/src/ir/properties/execution_data_state.rs +++ b/crates/types/src/ir/properties/execution_data_state.rs @@ -2,14 +2,9 @@ //! //! [`ExecutionTiming`] says when a node's value is produced (ingestion vs. //! query time); [`ExecutionDataState`] pairs it with the [`DataPrimitive`] -//! the edge carries (raw values vs. summary state). [`Materialization`] is -//! the per-sub-DAG choice a node's timing is derived from. The rules that assign -//! and check them over a DAG live in [`crate::ir::properties::timing`], which reports -//! violations as [`ExecutionDataStateError`]. - -use thiserror::Error; - -use crate::physical::Materialization; +//! the edge carries (raw values vs. summary state). A node's timing comes +//! from the [`crate::stages::PhysicalProperties`] of the physical candidate +//! it realizes. /// When a post-ASAP value is produced. #[derive( @@ -78,121 +73,6 @@ impl std::fmt::Display for ExecutionDataState { } } -/// A plan-construction-time data_state violation. Typed (not a string) so a -/// strategy can degrade to a conservative fallback on the specific variant -/// it expects, and so tests can assert the *reason* a plan was rejected. -#[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum ExecutionDataStateError { - #[error("invalid maintained-population maintenance/evaluation contract")] - InvalidMaintainedPopulation, - /// A query-time value (a `SummaryEstimate` or query-time operator output) - /// placed beneath a summary built at ingestion time — the one shape issue - /// #171's data_state split exists to make unrepresentable. - #[error( - "query-time value under an ingestion-time summary: {edge} received a {child} input, but a \ - summary built at ingestion time can only consume ingestion-time rows (or exact \ - accumulator state)" - )] - QueryTimeValueUnderIngestionSummary { - edge: &'static str, - child: ExecutionDataState, - }, - /// A summary build at query time over an input (rows or exact-accumulator - /// state) produced at ingestion time and not stored there (the input node - /// is not `MatAtIngestion`): at query time that input no longer exists. - /// Bad input (a hand-built or candidate-fixed plan), not an internal bug. - #[error( - "a query-time {edge} reads {operator} output produced at ingestion time that is not \ - stored (not MatAtIngestion)" - )] - UnstoredIngestionInputUnderQueryTimeSummary { - edge: &'static str, - operator: &'static str, - }, - /// Any other edge whose child data_state the parent does not accept - /// (e.g. plain update rows fed straight into a `SummaryEstimate`, or a - /// sketch's opaque state fed into a query-time operator). - #[error("{edge} does not accept a {child} input")] - IllegalChildDataState { - edge: &'static str, - child: ExecutionDataState, - }, - /// A `SummaryAgg` whose child is summary state of a family other than an - /// exact accumulator — re-accumulating opaque sketch/sample/… state on - /// the update path has no defined semantics here. - #[error( - "SummaryAgg.child carries {family} summary state; only exact accumulator state can be \ - composed into another maintained summary" - )] - UnsupportedStateComposition { family: String }, - /// One shared node assigned two different execution timings by its - /// consumers; no single execution of it can serve both. - #[error("shared node is assigned conflicting timings: {first} and {second}")] - ConflictingTiming { - first: ExecutionDataState, - second: ExecutionDataState, - }, - /// A plan root that runs at ingestion time without being stored there - /// (a timing fixed by a candidate builder): no query reads its rows. A - /// root assigned `MatAtIngestion` is valid; the query reads the stored - /// output. - #[error( - "{operator} root runs at ingestion time but is not MatAtIngestion: no query reads its \ - output" - )] - UnstoredIngestionRoot { operator: &'static str }, - /// An explicit materialization on a node whose timing a candidate builder - /// fixed (e.g. a query-time `FinalizeExactAccumulator` read boundary) - /// that needs the other timing. - #[error( - "{operator} is fixed at {} by its candidate; materialization {materialization:?} needs \ - the other timing", - fixed.as_str() - )] - MaterializationConflictsWithFixedTiming { - operator: &'static str, - fixed: ExecutionTiming, - materialization: Materialization, - }, - /// A summary build with a filter placed at ingestion time. An - /// ingestion-time build reads raw data samples, which carry none of the - /// query's columns the filter reads, so only a query-time build can - /// apply it. - #[error("{operator} has a filter, which only a query-time build can apply")] - FilteredSummaryAtIngestion { operator: &'static str }, - /// An ingestion-time `BinaryOp` the update path cannot evaluate: not - /// arithmetic without vector matching, operands that are not the same - /// per-series rows, or a schema other than non-null timestamps, at most - /// one series identity and one Float64 value. - #[error("unsupported ingestion-time binary schema or operator")] - InvalidIngestionTimeBinary, - #[error("checked division requires one valid guard on a read-time division operator")] - InvalidCheckedDivision, - /// An exact operator whose input columns are not all `Plain` at its - /// declared data_state. - #[error("exact operator consumes non-plain column {column:?} ({dtype})")] - NonPlainOperand { column: String, dtype: String }, - /// An ASAP operator with no execution timing contract yet in an - /// executable plan: the reserved `SummarySubtract`, `SummaryDelete`, - /// `SummaryJoin` and `Extension`. - #[error("{operator} has no execution timing contract yet")] - UnimplementedOperator { operator: &'static str }, - /// A node assigned `MatAtIngestion` over a sub-DAG that can only run at - /// query time (#509 Algorithm 3, line 13). - #[error("{operator}: upstream of ingestion-time node not at ingestion time")] - UpstreamNotAtIngestion { operator: &'static str }, - /// A node reached by export without a timing: the materialization timing pass - /// was not applied to the DAG first. - #[error( - "{operator} node has no execution timing; apply materialization timings before export" - )] - UntimedNode { operator: &'static str }, - /// The exported physical DAG failed its own validation, e.g. an empty - /// batch or a hand-timed plan the timing pass would have rejected. - #[error("invalid physical ASAP DAG: {0}")] - InvalidPhysicalDAG(#[from] crate::ir::physical_export::PhysicalASAPDAGValidationError), -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/types/src/ir/properties/mod.rs b/crates/types/src/ir/properties/mod.rs index 455a489f2..5eecc5878 100644 --- a/crates/types/src/ir/properties/mod.rs +++ b/crates/types/src/ir/properties/mod.rs @@ -1,28 +1,10 @@ //! #511 §2.2–2.3: the planning properties a node carries besides its //! operator and schema. //! -//! - [`accuracy`] — accuracy requirements and the error metrics they are -//! stated in ([`AccuracyTarget`], [`AccuracyMetric`]). -//! - [`guarantee`] — machine-readable accuracy guarantees -//! ([`AccuracyGuarantee`]). //! - [`execution_data_state`] — when a node's value is produced //! ([`ExecutionTiming`]) and what its edge carries ([`ExecutionDataState`]). -//! - [`timing`] — execution timing written into every node from a -//! materialization assignment, and its validation. //! - [`summary_coverage`] — what a summary state covers. -pub mod accuracy; pub mod execution_data_state; -pub mod guarantee; pub mod summary_coverage; -pub mod timing; -pub use accuracy::{ - default_metric_for_root, AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, NormScale, -}; -pub use execution_data_state::{ - DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, -}; -pub use guarantee::{ - AccuracyError, AccuracyGuarantee, BoundExpr, CompositionOperator, GuaranteeSource, - ProbabilityExpr, VALUES_PER_RESULT, -}; +pub use execution_data_state::{DataPrimitive, ExecutionDataState, ExecutionTiming}; diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 5cf1812ed..e67a5acc0 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -3,67 +3,82 @@ //! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after //! Goldstein & Larson's view matching. //! -//! Coverage is derived from the node, never declared. Walking down from a -//! `SummaryAgg`, a predicate conjunct moves into `selection` when it can be -//! lifted to the `SummaryAgg` (through `Filter`, a range `TimeRange`, a -//! `TimeShift` without `@` and direct-column `Project` items) and it is a -//! value set or an interval on one column. Everything else stays in -//! `definition` as a residual, so two states are merged only when they -//! compute the same thing over disjoint rows. +//! Coverage is derived once for each summary-state memo group, from its +//! input groups, never declared, and stored in its `GroupProperties`. +//! Walking down from a `SummaryAgg` through the input groups' expressions, a +//! predicate conjunct moves into `selection` when it can be lifted to the +//! `SummaryAgg` (through `Filter`, a range `TimeRange`, a `TimeShift` without +//! `@` and direct-column `Project` items) and it is a value set or an +//! interval on one column. Everything else stays in `definition` as a +//! residual, so two states are merged only when they compute the same thing +//! over disjoint rows. Node trees carry no coverage. use std::cmp::Ordering; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::BTreeMap; use std::ops::Bound; -use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::arguments::Reduction; +use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; +use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; +use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; -use crate::ir::operator::operator_properties::Reduction; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{Predicate, ScalarExpr}; -use crate::ir::schema::family_capability::{FamilyCapability, MergeInputRule}; -use crate::ir::schema::sketch::GroupingStrategy; use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::SchemaDerivationError; +use crate::stages::{MemoGroupAsInput, MemoGroupId}; +use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; + +/// What a summary state computes, with every lifted selection removed. +#[derive(Debug, Clone, PartialEq)] +pub struct SummaryStateDefinition { + /// The group the unlifted operators read: the first input under the + /// SummaryAgg that is not Filter / range TimeRange / TimeShift without @ + /// / direct-column Project. `None` when that input is a `Scan` with a + /// liftable predicate (a PromQL label matcher): it is lifted too, and the + /// `Scan` with the rest of its predicates is the last unlifted operator, + /// so states over one source with different matchers may merge. + pub input_group: Option, + /// The operators not lifted into the selection, listed top down from the + /// SummaryAgg, each with only its residual arguments. `None` marks an + /// operator's main input (the next unlifted operator, or input_group); + /// `Some(g)` is another input group it reads, such as the group a scalar + /// subquery in a Filter reads. Two states may merge only if input_group + /// and these operators are equal. + pub unlifted_operators: Vec>>, +} #[derive(Debug, Clone, PartialEq)] pub struct SummaryCoverage { - /// The `SummaryAgg` with the selection removed from its sub-DAG. - pub(crate) definition: Rc, - /// Union of boxes over the output rows of the definition's child. + pub(crate) definition: SummaryStateDefinition, + /// Union of boxes over the output rows of the unlifted operators' input. pub(crate) selection: Vec, } #[cfg(any(test, feature = "test-support"))] impl SummaryCoverage { - /// The `SummaryAgg` with the selection removed from its sub-DAG. - pub fn definition(&self) -> &Rc { + pub fn definition(&self) -> &SummaryStateDefinition { &self.definition } - /// Union of boxes over the output rows of the definition's child. + /// Union of boxes over the output rows of the unlifted operators' input. pub fn selection(&self) -> &[SelectionBox] { &self.selection } } -/// A conjunction of per-column constraints and an optional time window. -/// A column or time it does not mention is unrestricted. JSON: `columns` is -/// a list of `{column, constraint}` (its keys are not strings), and a bound -/// is `{"included": v}`, `{"excluded": v}` or `"unbounded"`. +/// A conjunction of per-column constraints. The timestamp column is one of +/// the columns; there is no separate time field. A column it does not +/// mention is unrestricted. JSON: `columns` is a list of +/// `{column, constraint}` (its keys are not strings), and a bound is +/// `{"included": v}`, `{"excluded": v}` or `"unbounded"`. #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SelectionBox { #[serde(with = "columns_as_list")] pub(crate) columns: BTreeMap, - /// Offsets from the evaluation time in milliseconds: a PromQL range - /// `TimeRange(w)` over `TimeShift(s)` is `(-(s + w), -s]`. Absolute - /// time will be an ordinary interval on the timestamp column once the IR - /// has timestamp literals. - #[serde(with = "relative_time_bounds")] - pub(crate) relative_time: Option<(Bound, Bound)>, } /// `SelectionBox::columns` as a list of `{column, constraint}`. @@ -133,33 +148,6 @@ impl From> for Bound { } } -/// `SelectionBox::relative_time` as `[lower, upper]` wire bounds. -mod relative_time_bounds { - use super::WireBound; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use std::ops::Bound; - - /// A relative time window, `(lower, upper)`. - type Window = (Bound, Bound); - - pub(super) fn serialize( - window: &Option, - serializer: S, - ) -> Result { - window - .as_ref() - .map(|(lower, upper)| (WireBound::from(lower), WireBound::from(upper))) - .serialize(serializer) - } - - pub(super) fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result, D::Error> { - let window = Option::<(WireBound, WireBound)>::deserialize(deserializer)?; - Ok(window.map(|(lower, upper)| (lower.into(), upper.into()))) - } -} - /// A column of the definition child's output, by `(table, name)`. A /// `Project` below renames its columns and sets their table to its /// qualifier (none by default). @@ -180,28 +168,58 @@ impl ColumnIdentity { } } +/// A constraint on one column of a selection. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum Constraint { + /// The value is one of these. In(Vec), + /// The value is none of these. NotIn(Vec), + /// The value lies between `lower` and `upper`; each end is `Included`, + /// `Excluded` or `Unbounded`. A PromQL range `m[r]` evaluated at t reads + /// the samples in (t − r, t]: left-open, right-closed. Examples, on the + /// timestamp column `ts` (ms) unless stated: + /// + /// | Query fragment | `lower` | `upper` | + /// |---|---|---| + /// | PromQL `m[5m]` | `Excluded(EvaluationTimePlusMs(-300_000))` | `Included(EvaluationTimePlusMs(0))` | + /// | PromQL `m[5m] offset 1h` | `Excluded(EvaluationTimePlusMs(-3_900_000))` | `Included(EvaluationTimePlusMs(-3_600_000))` | + /// | PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | + /// | SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(..)))` | `Unbounded` | + /// | SQL `WHERE price > 10` (column `price`) | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | Interval { - #[serde(with = "value_bound")] - lower: Bound, - #[serde(with = "value_bound")] - upper: Bound, + #[serde(with = "limit_bound")] + lower: Bound, + #[serde(with = "limit_bound")] + upper: Bound, }, } +/// An end of a [`Constraint::Interval`]. Two limits compare only when they +/// are of the same variant: a fixed value against a fixed value of the same +/// type, an offset from the evaluation time against another such offset. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IntervalConstraintLimit { + /// A constant: `Float64(10.0)` for SQL `price > 10`; + /// `Timestamp(1_700_000_000_000)` for PromQL `@ 1700000000` or a SQL + /// timestamp literal. + FixedValue(ScalarValue), + /// The evaluation time t of a repeatedly evaluated query plus this many + /// ms (negative: before t); only on the timestamp column. `m[5m]` is + /// `(EvaluationTimePlusMs(-300_000), EvaluationTimePlusMs(0)]`. + EvaluationTimePlusMs(i64), +} + /// A `Constraint::Interval` bound as a [`WireBound`]. -mod value_bound { - use super::WireBound; - use crate::ir::scalar::expr_ir::ScalarValue; +mod limit_bound { + use super::{IntervalConstraintLimit, WireBound}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::ops::Bound; pub(super) fn serialize( - bound: &Bound, + bound: &Bound, serializer: S, ) -> Result { WireBound::from(bound).serialize(serializer) @@ -209,7 +227,7 @@ mod value_bound { pub(super) fn deserialize<'de, D: Deserializer<'de>>( deserializer: D, - ) -> Result, D::Error> { + ) -> Result, D::Error> { Ok(WireBound::deserialize(deserializer)?.into()) } } @@ -223,11 +241,6 @@ pub enum CoverageError { "coverage is derived only for SummaryAgg, SummaryTumblingWindowChain and SummaryMerge" )] NotSummary, - /// A `SummaryTumblingWindowChain` that does not expand to one build over its whole - /// window (its window overflows, or the build does not derive): bad - /// input. - #[error("tumbling-window chain does not expand to one build over its whole window: {reason}")] - TumblingWindowChainExpansion { reason: String }, /// A `SummaryMerge` without inputs: bad input. #[error("summary merge requires at least one input")] EmptyMerge, @@ -250,127 +263,109 @@ pub enum CoverageError { /// none): bad input. #[error("regrouping merge key {key} is not a group key of its input states")] NotAGroupKey { key: ColumnId }, - /// A regrouping `SummaryMerge` of states whose family does not merge - /// different groups: counter states (`Rate`, `Increase`) merge - /// consecutive windows of one series, a family without a merge merges - /// nothing, and a Hydra's groups share one state. Bad input. - #[error("the inputs' family cannot merge different groups")] - FamilyCannotRegroup, + /// A regrouping `SummaryMerge` of states whose summary type does not + /// merge different groups: counter states (`Rate`, `Increase`) merge + /// consecutive windows of one series, a summary type without a merge + /// merges nothing, and a Hydra's groups share one state. Bad input. + #[error("the inputs' summary type cannot merge different groups")] + SummaryTypeCannotRegroup, /// The build a regrouping `SummaryMerge` stands for does not derive: /// bad input. #[error("the regrouped build does not derive: {reason}")] RegroupedBuild { reason: String }, } -/// Coverage already derived during one validation, by node address. The -/// nodes must stay alive while it is in use. -pub(crate) type NodeToSummaryCoverageCache = HashMap<*const OperatorNode, SummaryCoverage>; +/// A memo group read as an input, and the inputs a rule builds over such +/// groups. +type GroupInput<'m> = InputOrTemporaryBuild>; impl SummaryCoverage { - /// Coverage of a `SummaryAgg`, `SummaryTumblingWindowChain` or `SummaryMerge`. A chain - /// covers what one build over its whole window does, and a merge of its - /// newest tumbling windows what one build over their window does. A merge fails unless - /// every input has the same definition and their selections are + /// Coverage of a summary state a memo group computes with `operator` + /// over its input groups: a `SummaryAgg`, a `SummaryTumblingWindowChain`, + /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped + /// `SummaryAgg`. A merge reads its inputs' stored coverage and fails + /// unless every input has the same definition and their selections are /// pairwise disjoint; a family whose merge is idempotent - /// ([`MergeInputRule::MayOverlap`] in the family-capability - /// table) may also merge overlapping selections, and a counter family + /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may + /// also merge overlapping selections, and a counter family /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. - pub(crate) fn derive(node: &OperatorNode) -> Result { - Self::derive_cached(node, &mut NodeToSummaryCoverageCache::new()) - } - - /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, - /// of its sub-DAG's operators, output kinds and schemas, children - /// first, without planning metadata (`timing`, `guarantee`, - /// `materialization`). Two equal definitions (the equality a merge - /// requires) get equal ids on any node and in any process, so a summary - /// store can key states by this id and a concrete selection. - pub(crate) fn definition_id(&self) -> String { - format!( - "{:032x}", - definition_hash(&self.definition, &mut HashMap::new()) - ) - } - - /// [`derive`](Self::derive), reusing and recording the coverage of - /// every summary state in `cache`, so validating a DAG derives each - /// state once. - pub(crate) fn derive_cached( - node: &OperatorNode, - cache: &mut NodeToSummaryCoverageCache, + pub(crate) fn derive_from_inputs<'m>( + operator: &Operator>, ) -> Result { - let key = node as *const OperatorNode; - if let Some(coverage) = cache.get(&key) { - return Ok(coverage.clone()); - } - if let Some(build) = selected_build(node)? { - let coverage = of_summary_agg(&build); - cache.insert(key, coverage.clone()); - return Ok(coverage); - } - let coverage = match node.asap() { - Some(ASAPOp::SummaryAgg { .. }) => of_summary_agg(node), - // Back-to-back tumbling windows cover the whole window one build over it - // would. - Some(ASAPOp::SummaryTumblingWindowChain { .. }) => of_summary_agg( - &crate::ir::operator::tumbling_window_chain::whole_window_build(node).map_err( - |error| CoverageError::TumblingWindowChainExpansion { - reason: error.to_string(), - }, - )?, - ), - // The newest tumbling windows of one chain cover what one build over their - // window does. - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows: Some(_), - group_by: None, - .. - }) => of_summary_agg( - &crate::ir::operator::tumbling_window_chain::merged_window_build(node).map_err( - |error| CoverageError::TumblingWindowChainExpansion { - reason: error.to_string(), - }, - )?, + let as_built = |op: &ASAPOp>| -> ASAPOp> { + op.map_children(|input| InputOrTemporaryBuild::Input(*input)) + }; + match operator { + Operator::NonASAP(_) => match selected_build(operator)? { + Some(build) => of_summary_agg(&build), + None => Err(CoverageError::NotSummary), + }, + Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => of_summary_agg(&as_built(op)), + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => todo!( + "#1207 PR 2: a chain's coverage is a build's over its input, with the timestamp column unconstrained" ), - Some(ASAPOp::SummaryMerge { + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { let inputs = children .iter() - .map(|child| Self::derive_cached(child, cache)) + .map(|child| { + child + .aggregate_coverage() + .cloned() + .ok_or(CoverageError::NotSummary) + }) .collect::, _>>()?; let merged = of_merge(&inputs)?; match group_by { - None => merged, - Some(keys) => regrouped(merged, keys)?, + None => Ok(merged), + Some(keys) => regrouped(merged, &children[0], keys), } } - _ => return Err(CoverageError::NotSummary), - }; - cache.insert(key, coverage.clone()); - Ok(coverage) + Operator::ASAP(_) => Err(CoverageError::NotSummary), + } + } + + /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, + /// of its input group and unlifted operators. Two equal definitions + /// (the equality a merge requires) of one memo get equal ids, so a + /// summary store can key states by this id and a concrete selection. + pub(crate) fn definition_id(&self) -> String { + const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; + const PRIME: u128 = 0x0000000001000000000000000000013b; + let mut hash = OFFSET; + for byte in format!("{:?}", self.definition).bytes() { + hash ^= u128::from(byte); + hash = hash.wrapping_mul(PRIME); + } + format!("{hash:032x}") } } /// For a `Filter` that keeps whole states of a grouped `SummaryAgg` by /// their key columns, the build that computes exactly the kept states: the /// same build over its input rows filtered by the same predicate on the -/// input columns those keys name. `None` for any other node. -fn selected_build(node: &OperatorNode) -> Result>, CoverageError> { - let Some(states) = crate::ir::operator::asap::selected_from(node) else { +/// input columns those keys name. `None` for any other operator. +fn selected_build<'m>( + operator: &Operator>, +) -> Result>>, CoverageError> { + let Operator::NonASAP(NonASAPOp::Filter { + pred, + child: states, + }) = operator + else { return Ok(None); }; - let Some(NonASAPOp::Filter { pred, .. }) = node.non_asap() else { + if operator.output_kind() != OperatorResultKind::State { return Ok(None); - }; - let Some(ASAPOp::SummaryAgg { + } + let Operator::ASAP(ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, reduction: Reduction::Reduce(by), - grouping, filter, - }) = states.asap() + }) = states.operator() else { return Err(CoverageError::NotSummary); }; @@ -381,7 +376,7 @@ fn selected_build(node: &OperatorNode) -> Result>, Cover let mut unmapped = None; let rows_pred = pred.0.map_columns(&mut |column| { let mapped = by.keys().get(column).copied().filter(|&input_column| { - matches!((states.schema.fields.get(column), child.schema.fields.get(input_column)), + matches!((states.schema().fields.get(column), child.schema().fields.get(input_column)), (Some(at_state), Some(at_input)) if at_state.name == at_input.name) }); mapped.unwrap_or_else(|| { @@ -392,54 +387,54 @@ fn selected_build(node: &OperatorNode) -> Result>, Cover if let Some(key) = unmapped { return Err(CoverageError::NotAGroupKey { key }); } - let rows = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: crate::ir::scalar::Predicate(rows_pred), - child: Rc::clone(child), - })) - .map_err(|error| CoverageError::RegroupedBuild { + let mut built = |input: &MemoGroupAsInput<'m>| InputOrTemporaryBuild::Input(*input); + let regrouped_build = |error: SchemaDerivationError| CoverageError::RegroupedBuild { reason: error.to_string(), - })?; - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + }; + let rows = InputOrTemporaryBuild::build(NonASAPOp::Filter { + pred: Predicate(rows_pred.map_operator_refs(&mut built)), + child: InputOrTemporaryBuild::Input(child), + }) + .map_err(regrouped_build)?; + let reduction = Reduction::Reduce(by); + summary_agg_schema(rows.schema(), &family, &reduction).map_err(regrouped_build)?; + Ok(Some(ASAPOp::SummaryAgg { child: rows, - family: family.clone(), - input: input.clone(), - reduction: Reduction::Reduce(by.clone()), - grouping: grouping.clone(), - filter: filter.clone(), + summary_type: family, + input, + reduction, + filter: filter.map(|pred| Predicate(pred.0.map_operator_refs(&mut built))), })) - .map(Some) - .map_err(|error| CoverageError::RegroupedBuild { - reason: error.to_string(), - }) } /// The coverage of a merge of states covering `inputs`. fn of_merge(inputs: &[SummaryCoverage]) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; - let mut proven = HashSet::new(); - if !inputs + if inputs .iter() - .all(|input| same_definition(&first.definition, &input.definition, &mut proven)) + .any(|input| input.definition != first.definition) { return Err(CoverageError::DefinitionMismatch); } // A family without a merge is checked as the strictest rule; the // executor rejects the merge itself. - let required = match first.definition.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => FamilyCapability::of(family).merge_inputs, + let required = match first.definition.unlifted_operators.first() { + Some(Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + .. + })) => SummaryTypeCapability::of(family).merge_inputs, _ => None, } .unwrap_or(MergeInputRule::Disjoint); let separated: fn(&SelectionBox, &SelectionBox) -> bool = match required { MergeInputRule::MayOverlap => |_, _| true, - MergeInputRule::DisjointInTime => SelectionBox::disjoint_in_time, + MergeInputRule::DisjointInTime => todo!( + "#1207 PR 2: disjoint intervals on the timestamp column, with IntervalConstraintLimit ends" + ), MergeInputRule::Disjoint => SelectionBox::disjoint, }; - // Windows in time order, each ending before the next starts, are - // pairwise disjoint in time, and so pairwise disjoint: O(b log b) for - // b boxes. Otherwise, every pair of boxes of different inputs. + // Every pair of boxes of different inputs. if required != MergeInputRule::MayOverlap - && !windows_ordered_apart(inputs) && inputs.iter().enumerate().any(|(i, input)| { inputs[..i].iter().any(|other| { input @@ -452,35 +447,37 @@ fn of_merge(inputs: &[SummaryCoverage]) -> Result Result { - let Some(ASAPOp::SummaryAgg { - child, - family, +/// The coverage of a regrouping merge, keyed by `keys`, of states whose +/// merge covers `merged` (`first` is its first input): the same rows +/// (`selection`), computed by the input's build grouped by the columns +/// `keys` name. The input states of one output group are different groups +/// of one build, so their rows are disjoint. A counter family merges only +/// time-disjoint windows, never different groups at one time. +fn regrouped( + merged: SummaryCoverage, + first: &MemoGroupAsInput<'_>, + keys: &[ColumnId], +) -> Result { + let Some(Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, input, reduction, - grouping, filter, - }) = merged.definition.asap() + .. + })) = merged.definition.unlifted_operators.first() else { return Err(CoverageError::NotSummary); }; - if !FamilyCapability::merges_groups(family) - || *grouping != GroupingStrategy::PerSubpopulationInstance - { - return Err(CoverageError::FamilyCannotRegroup); + if !SummaryTypeCapability::merges_groups(family) { + return Err(CoverageError::SummaryTypeCannotRegroup); } - let state = &merged.definition.schema; + let state = first.schema(); + let rows = build_input_schema(first).ok_or(CoverageError::NotSummary)?; // The build's input column each key of the state names. let mapped = keys .iter() @@ -494,7 +491,7 @@ fn regrouped(merged: SummaryCoverage, keys: &[ColumnId]) -> Result Result, _>>()?; - // Derived as a direct build's schema, so its definition id is the same. - let definition = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: family.clone(), + let reduction = Reduction::by(mapped); + summary_agg_schema(&rows, family, &reduction).map_err(|error| { + CoverageError::RegroupedBuild { + reason: error.to_string(), + } + })?; + // A direct build's definition, so its definition id is the same. + let mut definition = merged.definition.clone(); + definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { + child: None, + summary_type: family.clone(), input: input.clone(), - reduction: Reduction::by(mapped), - grouping: grouping.clone(), + reduction, filter: filter.clone(), - })) - .map_err(|error| CoverageError::RegroupedBuild { - reason: error.to_string(), - })?; + }); Ok(SummaryCoverage { definition, selection: merged.selection, }) } -/// Whether the time windows of every box of `inputs` are pairwise apart: -/// sorted by start, each ends before the next starts. `false` when a box -/// has no time window or two windows overlap, boxes of one input included. -fn windows_ordered_apart(inputs: &[SummaryCoverage]) -> bool { - let mut windows = Vec::new(); - for selection_box in inputs.iter().flat_map(|input| &input.selection) { - match &selection_box.relative_time { - Some(window) => windows.push(window), - None => return false, - } - } - // A lower bound's position: unbounded first, then by value, an - // included value before the excluded one. - let start = |(lower, _): &&(Bound, Bound)| match lower { - Bound::Unbounded => (false, 0, false), - Bound::Included(at) => (true, *at, false), - Bound::Excluded(at) => (true, *at, true), - }; - windows.sort_by_key(start); - windows.windows(2).all(|pair| { - let ((_, upper), (lower, _)) = (pair[0], pair[1]); - ends_before(upper, lower, |a: &i64, b: &i64| Some(a.cmp(b))) - }) -} - -/// The hash [`SummaryCoverage::definition_id`] reports: each node's -/// operator without its children, output kind and schema (in their `Debug` -/// form, which every IR type derives and which is fixed for a given IR), -/// then its children's hashes in order. `cache` hashes a shared node once. -fn definition_hash( - node: &Rc, - cache: &mut HashMap<*const OperatorNode, u128>, -) -> u128 { - if let Some(hash) = cache.get(&Rc::as_ptr(node)) { - return *hash; +/// The schema of the rows the build of summary state `state` reads (its +/// `SummaryAgg`'s input; one build over a chain's window reads its range of +/// the chain's input, which has the input's schema). `None` when `state` +/// is no summary state. +fn build_input_schema(state: &I) -> Option { + if let Some(states) = selected_from(state) { + return build_input_schema(&states); } - const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; - const PRIME: u128 = 0x0000000001000000000000000000013b; - let mut hash = OFFSET; - let mut write = |bytes: &[u8]| { - for byte in bytes { - hash ^= u128::from(*byte); - hash = hash.wrapping_mul(PRIME); + match state.operator() { + Operator::ASAP( + ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, + ) => Some(child.schema().clone()), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { + build_input_schema(children.first()?) } - }; - let own = format!( - "{:?}|{:?}|{:?}", - node.operator.map_children(|_| ()), - node.result_kind, - node.schema - ); - write(own.as_bytes()); - for child in node.children() { - let child_hash = definition_hash(child, cache); - write(&child_hash.to_le_bytes()); - } - cache.insert(Rc::as_ptr(node), hash); - hash -} - -/// Structural equality ignoring planning metadata (`timing`, `guarantee`). -/// `proven` memoizes node pairs already found equal. -fn same_definition( - a: &Rc, - b: &Rc, - proven: &mut HashSet<(*const OperatorNode, *const OperatorNode)>, -) -> bool { - let key = (Rc::as_ptr(a), Rc::as_ptr(b)); - if Rc::ptr_eq(a, b) || proven.contains(&key) { - return true; - } - let (ac, bc) = (a.children(), b.children()); - let equal = a.operator.map_children(|_| ()) == b.operator.map_children(|_| ()) - && a.result_kind == b.result_kind - && a.schema == b.schema - && ac.len() == bc.len() - && ac - .iter() - .zip(&bc) - .all(|(x, y)| same_definition(x, y, proven)); - if equal { - proven.insert(key); + _ => None, } - equal } -fn of_summary_agg(node: &OperatorNode) -> SummaryCoverage { - let Some(ASAPOp::SummaryAgg { +/// The coverage of `build`, a `SummaryAgg` over memo groups (or over an +/// input a rule built over them): its operators walked down through the +/// liftable operators of its input groups' expressions. +fn of_summary_agg(build: &ASAPOp>) -> Result { + let ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, reduction, - grouping, filter, - }) = node.asap() + } = build else { unreachable!("called on a SummaryAgg"); }; - - // The chain a predicate can be lifted through, top to bottom, and the - // first node below it. - let mut chain: Vec<&Rc> = Vec::new(); - let mut base = child; - while let Some(op) = base.non_asap() { - let passes = match op { - // A rename outside a `Project` would be dropped with the node - // when everything is lifted out of it, so such a node stays. - NonASAPOp::Project { .. } => true, - _ if base.operator.output_schema().ok().as_ref() != Some(&base.schema) => false, - NonASAPOp::Filter { .. } => true, - NonASAPOp::TimeRange { kind, .. } => *kind == TimeRangeKind::Range, - NonASAPOp::TimeShift { shift, .. } => shift.at.is_none(), + // The operators a predicate can be lifted through, top to bottom, and + // the first input below them. A subquery's group stays named in the + // residual of the operator that reads it. + let mut chain: Vec>> = Vec::new(); + let mut base = child.clone(); + loop { + let operator = base.operator(); + let passes = match &operator { + Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at.is_none(), _ => false, }; if !passes { break; } - chain.push(base); - base = base.children()[0]; + // The main input is an operator's first child. + base = operator.children()[0].clone(); + chain.push(operator); } - // Time is lifted only from exactly one range window. - let lift_time = chain - .iter() - .filter(|link| matches!(link.non_asap(), Some(NonASAPOp::TimeRange { .. }))) - .count() - == 1; - - let top = &child.schema; + let InputOrTemporaryBuild::Input(input_group) = &base else { + unreachable!("a rule builds only liftable operators over memo groups"); + }; + let top = child.schema(); // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual // of `chain[d]` when it is a `Filter`. let mut lifted = BTreeMap::new(); let agg_filter = filter .as_ref() .and_then(|pred| residual(pred, top, &[], &mut lifted)); - let kept: Vec> = chain + let kept: Vec>>> = chain .iter() .enumerate() - .map(|(depth, link)| match link.non_asap() { - Some(NonASAPOp::Filter { pred, .. }) => { + .map(|(depth, link)| match link { + Operator::NonASAP(NonASAPOp::Filter { pred, .. }) => { residual(pred, top, &chain[..depth], &mut lifted) } _ => None, }) .collect(); - let scan_kept = match base.non_asap() { - Some(NonASAPOp::Scan { predicates, .. }) => Some( - predicates + // A `Scan`'s predicates (PromQL label matchers) are lifted too; when + // one is, the `Scan` with the rest of its predicates is the last + // unlifted operator, and there is no input group. + let scan = match base.operator() { + Operator::NonASAP(NonASAPOp::Scan { + source, + predicates, + schema, + }) => { + let lifted_above = lifted.clone(); + let kept = predicates .iter() .filter_map(|pred| residual(pred, top, &chain, &mut lifted)) - .collect::>(), - ), + .collect(); + (lifted != lifted_above).then_some(NonASAPOp::Scan { + source, + predicates: kept, + schema, + }) + } _ => None, }; - // Bottom-up: rebuild without what was lifted. A node whose input and - // operator are unchanged keeps its `Rc`. - let mut rebuilt = Rc::clone(base); - if let ( - Some(kept), - Some(NonASAPOp::Scan { - source, - predicates, - schema, - }), - ) = (scan_kept, base.non_asap()) - { - if kept != *predicates { - rebuilt = rebuild( - base, - NonASAPOp::Scan { - source: source.clone(), - predicates: kept, - schema: schema.clone(), - }, - ); - } - } - let (mut window_ms, mut shift_ms) = (0i64, 0i64); - for (depth, link) in chain.iter().enumerate().rev() { - let child = Rc::clone(&rebuilt); - let op = match link.non_asap().expect("chain nodes are non-ASAP") { - NonASAPOp::Filter { pred, .. } => match &kept[depth] { - None => continue, - Some(rest) if rest == pred && Rc::ptr_eq(&child, link.children()[0]) => { - rebuilt = Rc::clone(link); - continue; + let mut unlifted_operators = vec![Operator::ASAP(ASAPOp::SummaryAgg { + child: None, + summary_type: family.clone(), + input: input.clone(), + reduction: reduction.clone(), + filter: agg_filter, + })]; + for (depth, link) in chain.iter().enumerate() { + match link { + Operator::NonASAP(NonASAPOp::Filter { .. }) => { + if let Some(rest) = kept[depth].clone() { + unlifted_operators.push(Operator::NonASAP(NonASAPOp::Filter { + pred: rest, + child: None, + })); } - Some(rest) => NonASAPOp::Filter { - pred: rest.clone(), - child, - }, - }, - NonASAPOp::TimeRange { range, .. } if lift_time => { - window_ms = range.as_millis() as i64; - continue; } - NonASAPOp::TimeShift { shift, .. } if lift_time => { - shift_ms += shift.offset_ms; - continue; - } - _ if Rc::ptr_eq(&child, link.children()[0]) => { - rebuilt = Rc::clone(link); - continue; - } - NonASAPOp::TimeRange { range, kind, .. } => NonASAPOp::TimeRange { - range: *range, - kind: *kind, - child, - }, - NonASAPOp::TimeShift { shift, .. } => NonASAPOp::TimeShift { - shift: *shift, - child, - }, - NonASAPOp::Project { - cols, qualifier, .. - } => NonASAPOp::Project { - cols: cols.clone(), - qualifier: qualifier.clone(), - child, - }, - _ => unreachable!("only liftable operators are on the chain"), - }; - rebuilt = rebuild(link, op); + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) => todo!( + "#1207 PR 2: lift the range and the offset into an interval on the timestamp column, with IntervalConstraintLimit ends" + ), + other => unlifted_operators.push(unlifted(other)), + } } - let definition = rebuild_asap( - node, - ASAPOp::SummaryAgg { - child: rebuilt, - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: agg_filter, - }, - ); + let input_group = match scan { + Some(scan) => { + unlifted_operators.push(Operator::NonASAP(scan)); + None + } + None => Some(input_group.group_id), + }; let selection = SelectionBox { columns: lifted @@ -771,45 +662,29 @@ fn of_summary_agg(node: &OperatorNode) -> SummaryCoverage { (identity, constraint) }) .collect(), - relative_time: lift_time.then_some(( - Bound::Excluded(-(shift_ms + window_ms)), - Bound::Included(-shift_ms), - )), }; - SummaryCoverage { - definition, + Ok(SummaryCoverage { + definition: SummaryStateDefinition { + input_group, + unlifted_operators, + }, selection: vec![selection], - } -} - -/// A node with the same output schema over a new operator. The definition -/// describes what was computed; it is not re-validated as a plan. -fn rebuild(original: &OperatorNode, op: NonASAPOp) -> Rc { - Rc::new(OperatorNode::with_schema( - Operator::NonASAP(op), - original.schema.clone(), - )) -} - -fn rebuild_asap(original: &OperatorNode, op: ASAPOp) -> Rc { - Rc::new(OperatorNode::with_schema( - Operator::ASAP(op), - original.schema.clone(), - )) + }) } /// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of -/// `top`, the agg child's schema) and return the rest. `above` are the chain -/// nodes between the predicate and the `SummaryAgg`, top to bottom. A column -/// whose `(table, name)` is not unique in `top` cannot be named in a -/// selection, so its conjuncts stay. +/// `top`, the agg child's schema) and return the rest, each scalar subquery +/// naming the group it reads. `above` +/// are the operators between the predicate and the `SummaryAgg`, top to +/// bottom. A column whose `(table, name)` is not unique in `top` cannot be +/// named in a selection, so its conjuncts stay. fn residual( - pred: &Predicate, + pred: &Predicate>, top: &Schema, - above: &[&Rc], + above: &[Operator>], lifted: &mut BTreeMap, -) -> Option { - let mut rest: Vec = Vec::new(); +) -> Option>> { + let mut rest: Vec>> = Vec::new(); for conjunct in pred.0.conjuncts() { let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; @@ -832,7 +707,7 @@ fn residual( Some((column, constraint)) => { lifted.insert(column, constraint); } - None => rest.push(conjunct.clone()), + None => rest.push(conjunct.map_operator_refs(&mut group_id)), } } match rest.len() { @@ -842,14 +717,38 @@ fn residual( } } +/// The memo group `input` is; `None` for a rule's temporary build. +fn group_id(input: &GroupInput<'_>) -> Option { + match input { + InputOrTemporaryBuild::Input(group) => Some(group.group_id), + InputOrTemporaryBuild::TemporaryBuild(_) => None, + } +} + +/// `link`, a liftable operator kept whole in a definition: its main input +/// `None`, every other input the group it is. +fn unlifted(link: &Operator>) -> Operator> { + let mut operator = link.map_children(group_id); + match &mut operator { + Operator::NonASAP( + NonASAPOp::Filter { child, .. } + | NonASAPOp::Project { child, .. } + | NonASAPOp::TimeRange { child, .. } + | NonASAPOp::TimeShift { child, .. }, + ) => *child = None, + _ => unreachable!("only liftable operators are walked"), + } + operator +} + /// Map a column of the input of `above.last()` up to the agg child's output: /// a `Project` passes it only as a direct column item. -fn column_at_top(mut column: ColumnId, above: &[&Rc]) -> Option { +fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option { for link in above.iter().rev() { - if let Some(NonASAPOp::Project { cols, .. }) = link.non_asap() { + if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { column = cols .iter() - .position(|item| item.expr == ScalarExpr::Column(column))?; + .position(|item| matches!(item.expr, ScalarExpr::Column(c) if c == column))?; } } Some(column) @@ -857,8 +756,8 @@ fn column_at_top(mut column: ColumnId, above: &[&Rc]) -> Option`, `>=`, `[NOT] IN (...)` and `OR` of /// equalities on one column, with non-null literals. -fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { - let literal = |e: &ScalarExpr| match e { +fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { + let literal = |e: &ScalarExpr| match e { ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), _ => None, }; @@ -931,8 +830,12 @@ fn flip(op: &CompareOpKind) -> Option { }) } +/// An interval between two fixed values. fn interval(lower: Bound, upper: Bound) -> Constraint { - Constraint::Interval { lower, upper } + Constraint::Interval { + lower: lower.map(IntervalConstraintLimit::FixedValue), + upper: upper.map(IntervalConstraintLimit::FixedValue), + } } fn dedup(values: Vec) -> Vec { @@ -975,10 +878,37 @@ fn compare(a: &ScalarValue, b: &ScalarValue) -> Option { (ScalarValue::Float64(a), ScalarValue::Float64(b)) => a.partial_cmp(b), (ScalarValue::Utf8(a), ScalarValue::Utf8(b)) => Some(a.cmp(b)), (ScalarValue::Boolean(a), ScalarValue::Boolean(b)) => Some(a.cmp(b)), + (ScalarValue::Timestamp(a), ScalarValue::Timestamp(b)) => Some(a.cmp(b)), _ => None, } } +/// Order of two interval limits; `None` across variants or when the fixed +/// values do not compare. +fn compare_limits(a: &IntervalConstraintLimit, b: &IntervalConstraintLimit) -> Option { + use IntervalConstraintLimit::*; + match (a, b) { + (FixedValue(a), FixedValue(b)) => compare(a, b), + (EvaluationTimePlusMs(a), EvaluationTimePlusMs(b)) => Some(a.cmp(b)), + _ => None, + } +} + +/// Whether value `v` lies inside the interval; `None` when unsure. +fn value_inside( + v: &ScalarValue, + bounds: ( + &Bound, + &Bound, + ), +) -> Option { + inside( + &IntervalConstraintLimit::FixedValue(v.clone()), + bounds, + compare_limits, + ) +} + /// Whether `v` lies inside the interval; `None` when it cannot be compared. fn inside( v: &T, @@ -1023,18 +953,18 @@ fn intervals_disjoint( /// The tighter of two lower (`want = Greater`) or upper (`Less`) bounds. fn tighter( - a: &Bound, - b: &Bound, + a: &Bound, + b: &Bound, want: Ordering, -) -> Option> { - let value = |bound: &Bound| match bound { +) -> Option> { + let value = |bound: &Bound| match bound { Bound::Included(v) | Bound::Excluded(v) => Some(v.clone()), Bound::Unbounded => None, }; Some(match (value(a), value(b)) { (None, _) => b.clone(), (_, None) => a.clone(), - (Some(x), Some(y)) => match compare(&x, &y)? { + (Some(x), Some(y)) => match compare_limits(&x, &y)? { Ordering::Equal if matches!(a, Bound::Excluded(_)) => a.clone(), Ordering::Equal => b.clone(), order if order == want => a.clone(), @@ -1055,7 +985,7 @@ impl Constraint { (In(a), Interval { lower, upper }) | (Interval { lower, upper }, In(a)) => { let mut kept = Vec::new(); for v in a { - if inside(v, (lower, upper), compare)? { + if value_inside(v, (lower, upper))? { kept.push(v.clone()); } } @@ -1086,7 +1016,7 @@ impl Constraint { (In(a), NotIn(b)) | (NotIn(b), In(a)) => a.iter().all(|v| among(v, b) == Some(true)), (In(a), Interval { lower, upper }) | (Interval { lower, upper }, In(a)) => a .iter() - .all(|v| inside(v, (lower, upper), compare) == Some(false)), + .all(|v| value_inside(v, (lower, upper)) == Some(false)), ( Interval { lower: al, @@ -1096,7 +1026,7 @@ impl Constraint { lower: bl, upper: bu, }, - ) => intervals_disjoint((al, au), (bl, bu), compare), + ) => intervals_disjoint((al, au), (bl, bu), compare_limits), (NotIn(_), _) | (_, NotIn(_)) => false, } } @@ -1104,57 +1034,30 @@ impl Constraint { #[cfg(any(test, feature = "test-support"))] impl SelectionBox { - pub fn new( - columns: BTreeMap, - relative_time: Option<(Bound, Bound)>, - ) -> Self { - Self { - columns, - relative_time, - } + pub fn new(columns: BTreeMap) -> Self { + Self { columns } } /// The per-column constraints. pub fn columns(&self) -> &BTreeMap { &self.columns } - - /// The time window, as offsets from the evaluation time in milliseconds. - pub fn relative_time(&self) -> Option<(Bound, Bound)> { - self.relative_time - } } impl SelectionBox { - /// Whether no row lies in both boxes: some column, or the time window, - /// is restricted by both and the restrictions are disjoint. + /// Whether no row lies in both boxes: some column is restricted by both + /// and the restrictions are disjoint. fn disjoint(&self, other: &Self) -> bool { - self.disjoint_in_time(other) - || self - .columns - .iter() - .any(|(column, a)| other.columns.get(column).is_some_and(|b| a.disjoint(b))) - } - - /// Whether the boxes' time windows do not overlap. - fn disjoint_in_time(&self, other: &Self) -> bool { - match (&self.relative_time, &other.relative_time) { - (Some((al, au)), Some((bl, bu))) => { - intervals_disjoint((al, au), (bl, bu), |a: &i64, b: &i64| Some(a.cmp(b))) - } - _ => false, - } + self.columns + .iter() + .any(|(column, a)| other.columns.get(column).is_some_and(|b| a.disjoint(b))) } } -/// Join boxes that differ only in one dimension whose union is again one -/// constraint: touching time windows or value ranges, or the values of one -/// `In` column. Gaps stay separate boxes. -fn union(boxes: Vec) -> Vec { - union_ordered_time_boxes(&boxes).unwrap_or_else(|| union_general(boxes)) -} - -fn union_general(mut boxes: Vec) -> Vec { +/// Join boxes that differ only in one column whose union is again one +/// constraint: touching value ranges, or the values of one `In` column. +/// Gaps stay separate boxes. +fn union(mut boxes: Vec) -> Vec { let mut i = 0; while i < boxes.len() { let joined = (i + 1..boxes.len()).find_map(|j| join(&boxes[i], &boxes[j]).map(|b| (j, b))); @@ -1170,75 +1073,11 @@ fn union_general(mut boxes: Vec) -> Vec { boxes } -/// Tumbling-window coverage has identical column predicates and pairwise disjoint time -/// intervals. Sort once and join adjacent intervals, rather than removing one -/// element from a large vector for every tumbling window. General multidimensional or -/// overlapping coverage continues through the existing exact union algorithm. -fn union_ordered_time_boxes(boxes: &[SelectionBox]) -> Option> { - let first = boxes.first()?; - if !boxes - .iter() - .all(|b| b.columns == first.columns && b.relative_time.is_some()) - { - return None; - } - if !boxes - .iter() - .all(|b| match b.relative_time.as_ref().unwrap() { - (Bound::Unbounded, _) | (_, Bound::Unbounded) => true, - (Bound::Included(lower), Bound::Included(upper)) => lower <= upper, - ( - Bound::Included(lower) | Bound::Excluded(lower), - Bound::Included(upper) | Bound::Excluded(upper), - ) => lower < upper, - }) - { - return None; - } - let mut order: Vec<_> = (0..boxes.len()).collect(); - order.sort_by_key( - |&index| match boxes[index].relative_time.as_ref().unwrap().0 { - Bound::Unbounded => (false, 0, false), - Bound::Included(at) => (true, at, false), - Bound::Excluded(at) => (true, at, true), - }, - ); - if !order.windows(2).all(|pair| { - let (_, upper) = boxes[pair[0]].relative_time.as_ref().unwrap(); - let (lower, _) = boxes[pair[1]].relative_time.as_ref().unwrap(); - ends_before(upper, lower, |a: &i64, b: &i64| Some(a.cmp(b))) - }) { - return None; - } - let mut merged: Vec<(usize, SelectionBox)> = Vec::new(); - for index in order { - if let Some((first_index, previous)) = merged.last_mut() { - if let Some(joined) = join(previous, &boxes[index]) { - *first_index = (*first_index).min(index); - *previous = joined; - continue; - } - } - merged.push((index, boxes[index].clone())); - } - // The general union retains the earliest occurrence of each joined box. - merged.sort_by_key(|(index, _)| *index); - Some(merged.into_iter().map(|(_, selection)| selection).collect()) -} - fn join(a: &SelectionBox, b: &SelectionBox) -> Option { if a == b { return Some(a.clone()); } - if a.columns == b.columns { - let ((al, au), (bl, bu)) = (a.relative_time.as_ref()?, b.relative_time.as_ref()?); - let time = touching((al, au), (bl, bu), |x: &i64, y: &i64| Some(x.cmp(y)))?; - return Some(SelectionBox { - columns: a.columns.clone(), - relative_time: Some(time), - }); - } - if a.relative_time != b.relative_time || a.columns.len() != b.columns.len() { + if a.columns.len() != b.columns.len() { return None; } let mut differing = a @@ -1263,17 +1102,14 @@ fn join(a: &SelectionBox, b: &SelectionBox) -> Option { upper: bu, }, ) => { - let (lower, upper) = touching((al, au), (bl, bu), compare)?; + let (lower, upper) = touching((al, au), (bl, bu), compare_limits)?; Constraint::Interval { lower, upper } } _ => return None, }; let mut columns = a.columns.clone(); columns.insert(column.clone(), joined); - Some(SelectionBox { - columns, - relative_time: a.relative_time, - }) + Some(SelectionBox { columns }) } /// The union of two intervals when one ends exactly where the other starts, @@ -1299,94 +1135,76 @@ fn touching( } #[cfg(test)] -mod tumbling_window_union_tests { +mod tests { use super::*; + use crate::ir::scalar::ExprSemantics; + use IntervalConstraintLimit::{EvaluationTimePlusMs, FixedValue}; - fn tumbling_window(start: i64, end: i64) -> SelectionBox { - SelectionBox { - relative_time: Some((Bound::Excluded(start), Bound::Included(end))), - ..Default::default() + fn relative(lower_ms: i64, upper_ms: i64) -> Constraint { + Constraint::Interval { + lower: Bound::Excluded(EvaluationTimePlusMs(lower_ms)), + upper: Bound::Included(EvaluationTimePlusMs(upper_ms)), } } + /// SQL `price > 10` lifts to `(FixedValue(10), ∞)`. #[test] - fn large_reverse_ordered_tumbling_window_union_is_one_exact_interval() { - let tumbling_windows: Vec<_> = (0..75_000) - .rev() - .map(|i| tumbling_window(i, i + 1)) - .collect(); - assert_eq!(union(tumbling_windows), vec![tumbling_window(0, 75_000)]); + fn a_value_comparison_is_an_interval_between_fixed_values() { + let price_gt_10: ScalarExpr<()> = ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), + semantics: ExprSemantics::Sql, + }; + assert_eq!( + constraint_of(&price_gt_10), + Some(( + 0, + Constraint::Interval { + lower: Bound::Excluded(FixedValue(ScalarValue::Float64(10.0))), + upper: Bound::Unbounded, + } + )) + ); } + /// `m[5m]` and `m[5m] offset 5m` are disjoint and touch; neither + /// compares with an `@ T` interval. #[test] - fn interval_fast_path_matches_general_union_for_permutations() { - use Bound::{Excluded as E, Included as I, Unbounded as U}; - for intervals in [ - [(U, I(0)), (E(0), I(1)), (E(1), I(2)), (E(2), U)], - [(E(0), E(1)), (I(1), I(1)), (E(1), I(2)), (E(4), I(5))], - [(E(0), I(1)), (E(3), E(4)), (I(4), I(5)), (E(5), I(6))], - ] { - let boxes: Vec<_> = intervals - .into_iter() - .map(|relative_time| SelectionBox { - relative_time: Some(relative_time), - ..Default::default() - }) - .collect(); - for a in 0..4 { - for b in 0..4 { - for c in 0..4 { - for d in 0..4 { - let order = [a, b, c, d]; - if (0..4).any(|i| order[..i].contains(&order[i])) { - continue; - } - let shuffled: Vec<_> = - order.into_iter().map(|i| boxes[i].clone()).collect(); - assert_eq!(union(shuffled.clone()), union_general(shuffled)); - } - } - } - } - } - let empty = SelectionBox { - relative_time: Some((E(0), E(0))), - ..Default::default() + fn limits_compare_only_within_one_variant() { + let last_5m = relative(-300_000, 0); + let offset_5m = relative(-600_000, -300_000); + assert!(last_5m.disjoint(&offset_5m)); + let at = Constraint::Interval { + lower: Bound::Excluded(FixedValue(ScalarValue::Timestamp(1_699_999_700_000))), + upper: Bound::Included(FixedValue(ScalarValue::Timestamp(1_700_000_000_000))), }; - let singleton = SelectionBox { - relative_time: Some((I(0), I(0))), - ..Default::default() + assert!(!last_5m.disjoint(&at)); + assert_eq!(last_5m.intersect(&at), None); + let column = ColumnIdentity::new(None, "ts"); + let one = |constraint: Constraint| SelectionBox { + columns: BTreeMap::from([(column.clone(), constraint)]), }; - let unusual = vec![empty.clone(), empty, singleton]; - assert!(union_ordered_time_boxes(&unusual).is_none()); - assert_eq!(union(unusual.clone()), union_general(unusual)); + assert_eq!( + union(vec![one(last_5m), one(offset_5m)]), + vec![one(relative(-600_000, 0))] + ); } #[test] - fn sorted_union_preserves_gaps_boundaries_and_original_group_order() { - assert_eq!( - union(vec![ - tumbling_window(10, 11), - tumbling_window(1, 2), - tumbling_window(0, 1), - tumbling_window(11, 12) - ]), - vec![tumbling_window(10, 12), tumbling_window(0, 2)] - ); - let missing_endpoint = SelectionBox { - relative_time: Some((Bound::Excluded(0), Bound::Excluded(1))), - ..Default::default() - }; + fn an_interval_round_trips_through_json() { + let constraint = relative(-3_900_000, -3_600_000); + let json = serde_json::to_value(&constraint).unwrap(); assert_eq!( - union(vec![missing_endpoint.clone(), tumbling_window(1, 2)]), - vec![missing_endpoint, tumbling_window(1, 2)] + json, + serde_json::json!({"interval": { + "lower": {"excluded": {"evaluation_time_plus_ms": -3_900_000}}, + "upper": {"included": {"evaluation_time_plus_ms": -3_600_000}}, + }}) ); - let overlaps = vec![tumbling_window(0, 2), tumbling_window(1, 3)]; - assert!(union_ordered_time_boxes(&overlaps).is_none()); - assert_eq!(union(overlaps.clone()), overlaps); assert_eq!( - union(vec![tumbling_window(0, 1), tumbling_window(0, 1)]), - vec![tumbling_window(0, 1)] + serde_json::from_value::(json).unwrap(), + constraint ); } } diff --git a/crates/types/src/ir/properties/timing.rs b/crates/types/src/ir/properties/timing.rs deleted file mode 100644 index 07684c9fc..000000000 --- a/crates/types/src/ir/properties/timing.rs +++ /dev/null @@ -1,1640 +0,0 @@ -//! Execution timing: written into every node from a materialization -//! assignment, then validated against each operator's kind and its consuming -//! edges. -//! -//! The logical DAG carries no timing. Materialization (#509 -//! §Materialization) gives each sub-DAG one of its options: -//! `NotMaterialized`, `MatAtIngestion(medium)`, `MatAtQuery(medium)` or -//! `MatAtQueryKeptAcrossEvaluations(medium)`. -//! [`MaterializationAssignment`] records that option per sub-DAG, named by -//! the node whose output would be stored (any node: a summary state, a -//! population, or e.g. the tumbling windows of #509 Example 4), and -//! [`apply_materialization_timings`] expands it into a timing on every node: -//! -//! - `SummaryEstimate` and `EvaluatePopulation` run at query time; -//! - a node assigned `MatAtIngestion` runs at ingestion time, and one -//! assigned `MatAtQuery` or `MatAtQueryKeptAcrossEvaluations` at query -//! time. A `SummaryAgg` or -//! `MaintainPopulation` without an assignment is `NotMaterialized` and runs -//! at query time. Every node upstream of an ingestion-time node also runs -//! at ingestion time, so `MatAtIngestion` (or an ingestion timing fixed by -//! a candidate builder) over something that can only exist at query time -//! is an error ([`ExecutionDataStateError::UpstreamNotAtIngestion`]); -//! - a PromQL `BinaryOp` over per-series inputs that are not the same series -//! rows runs at query time, since maintenance cannot align them; -//! - every other node is `NotMaterialized` and runs when its consumer runs: -//! everything that feeds an ingestion-time node runs at ingestion time, -//! everything above an evaluation at query time. -//! -//! A node whose timing a candidate builder fixed (e.g. the query-time -//! `FinalizeExactAccumulator` of `finalize_query_candidate`) keeps that -//! timing. An explicit assignment on it that agrees is kept; one that needs -//! the other timing is an error -//! ([`ExecutionDataStateError::MaterializationConflictsWithFixedTiming`]). -//! A root may be `MatAtIngestion`: the query reads its stored output. A root -//! of rows that runs at ingestion time without being stored there is an -//! error ([`ExecutionDataStateError::UnstoredIngestionRoot`]). -//! -//! A node reached from two consumers that need different timings cannot be -//! executed once for both; Stage 2's `split_shared_by_timing` copies such a -//! sub-DAG for one side (by [`timing_under_consumer`]) before the assignment -//! is applied, and the pass itself -//! rejects a conflict it still finds. -//! -//! ## Edge rules (checked after the write) -//! -//! | Consumer | Accepts from an input | -//! |---|---| -//! | `SummaryAgg.child`, `SummaryTumblingWindowChain.child` | Rows, or exact-accumulator state, never a query-time value when the state is maintained; a query-time build reads an ingestion-time input (rows or exact-accumulator state) only as a `MatAtIngestion` input's stored output | -//! | `SummaryEstimate.summary_input` | Summary state at either timing | -//! | `SummaryMerge.children` | Summary state at either timing; at ingestion time only for a merge at ingestion time | -//! | `FinalizeExactAccumulator.child` | Exact-accumulator state | -//! | `EvaluatePopulation.child` | A timed `MaintainPopulation` | -//! | `MaintainPopulation.child` | Rows at its own timing matching the population's input; unlike a `NonASAP` consumer, a query-time population does not read a `MatAtIngestion` input's stored output | -//! | any `NonASAP` consumer | Rows (or exact-accumulator state for a projection-like operator) at the consumer's own timing, or a `MatAtIngestion` output read at query time; ingestion work never reads a query-time value | - -use std::collections::HashMap; -use std::rc::Rc; - -use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; -use crate::ir::operator::non_asap::NonASAPOp; -use crate::ir::operator::operator_properties::BinaryOpKind; -use crate::ir::operator::query::QueryRoot; -use crate::ir::properties::execution_data_state::{ - DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, -}; -use crate::ir::schema::{DataType, FieldDataType, Schema}; -use crate::physical::materialization::Materialization; -#[cfg(any(test, feature = "test-support"))] -use crate::physical::materialization::Medium; - -/// The materialization option of each sub-DAG (#509 §Materialization: -/// "Materialization decides, for each sub-DAG, whether its output is -/// stored"). A sub-DAG is named by the node whose output would be stored (by -/// identity); any node may be assigned. A node without an assignment is not -/// stored: a `SummaryAgg` or `MaintainPopulation` takes the default, every -/// other node runs when its consumer runs. `Default` assigns -/// `NotMaterialized` to every sub-DAG. -/// -/// The assignment keeps every assigned node alive, so a later allocation -/// cannot reuse its address and match a stale entry. -#[derive(Clone, Default)] -pub struct MaterializationAssignment { - assigned: HashMap<*const OperatorNode, (Rc, Materialization)>, - unassigned: Materialization, -} - -/// Each assigned node as its operator kind and address, not its whole -/// sub-DAG. -impl std::fmt::Debug for MaterializationAssignment { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let assigned: Vec<_> = self - .assigned - .iter() - .map(|(ptr, (node, materialization))| { - ( - format!("{}@{ptr:p}", node.operator.kind_name()), - materialization, - ) - }) - .collect(); - f.debug_struct("MaterializationAssignment") - .field("assigned", &assigned) - .field("unassigned", &self.unassigned) - .finish() - } -} - -impl MaterializationAssignment { - /// Every `SummaryAgg` or `MaintainPopulation` not [`set`](Self::set) - /// explicitly takes `materialization` where the DAG allows it: a - /// `MatAtIngestion` default over a sub-DAG that can only run at query - /// time leaves that sub-DAG `NotMaterialized`. - pub fn with_default(materialization: Materialization) -> Self { - Self { - assigned: HashMap::new(), - unassigned: materialization, - } - } - - /// Assign `materialization` to the sub-DAG whose output `node` produces. - pub fn set(&mut self, node: &Rc, materialization: Materialization) { - self.assigned - .insert(Rc::as_ptr(node), (Rc::clone(node), materialization)); - } - - /// The explicit assignment of `node`, if any. - pub fn get(&self, node: &OperatorNode) -> Option { - self.assigned - .get(&(node as *const OperatorNode)) - .map(|(_, m)| *m) - } - - /// The option `node` gets when `upstream_at_query_time` says whether - /// something below it can only run at query time. - fn resolve( - &self, - node: &OperatorNode, - upstream_at_query_time: bool, - ) -> Result { - let at_ingestion = |m: Materialization| m.timing() == ExecutionTiming::IngestionTime; - match self.get(node) { - Some(m) if at_ingestion(m) && is_filtered_build(node) => { - Err(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: node.operator.kind_name(), - }) - } - Some(m) if at_ingestion(m) && upstream_at_query_time => { - Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }) - } - Some(m) => Ok(m), - None if at_ingestion(self.unassigned) - && (upstream_at_query_time || is_filtered_build(node)) => - { - Ok(Materialization::NotMaterialized) - } - None => Ok(self.unassigned), - } - } -} - -/// Whether `operator`'s output is a sub-DAG output that materialization may -/// store: a `SummaryAgg`, a `SummaryTumblingWindowChain` or a `MaintainPopulation`. -pub(crate) fn is_materialization_point(operator: &Operator) -> bool { - matches!( - operator, - Operator::ASAP( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryTumblingWindowChain { .. } - | ASAPOp::MaintainPopulation { .. } - ) - ) -} - -/// Whether `node` is a summary build (`SummaryAgg` or `SummaryTumblingWindowChain`) with a -/// filter, which never runs at ingestion time -/// ([`ExecutionDataStateError::FilteredSummaryAtIngestion`]). -pub fn is_filtered_build(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - filter: Some(_), - .. - } | ASAPOp::SummaryTumblingWindowChain { - filter: Some(_), - .. - } - ) - ) -} - -/// Cache of one [`apply_materialization_timings`] pass: `input node → timed node`, -/// shared by every root of a workload so a node shared by two roots stays -/// one `Rc`. Re-reaching a node with a different timing is a conflict. -#[derive(Default)] -pub struct NodeToTimedNodeCache { - timed_node_of: HashMap<*const OperatorNode, Rc>, -} - -impl NodeToTimedNodeCache { - pub fn new() -> Self { - Self::default() - } - - /// The timed node produced for `input`, if the pass has reached it. - pub(crate) fn timed(&self, input: &Rc) -> Option<&Rc> { - self.timed_node_of.get(&Rc::as_ptr(input)) - } -} - -/// What `node`'s output carries: summary state from a build, a merge or a -/// node that keeps whole states (a key filter, a projection of the state -/// column), raw rows otherwise. -fn primitive(node: &OperatorNode) -> DataPrimitive { - match &node.operator { - Operator::ASAP(op) if op.produced_state().is_some() => DataPrimitive::SummaryState, - Operator::ASAP(ASAPOp::MaintainPopulation { .. }) => DataPrimitive::SummaryState, - _ if node.result_kind == crate::ir::OperatorResultKind::State => { - DataPrimitive::SummaryState - } - _ => DataPrimitive::Raw, - } -} - -/// The data state a timed node's output carries. -pub fn data_state(node: &OperatorNode) -> Option { - Some(ExecutionDataState { - timing: node.timing?, - primitive: primitive(node), - }) -} - -/// Whether the sub-DAG below `node` contains a node that can only run at -/// query time (an evaluation, or a sub-DAG `assignment` does not put at -/// ingestion time), which forces every consumer above it to query time as well. -/// A PromQL `BinaryOp` whose inputs are not the same per-series rows also runs -/// at query time. -fn forces_query_time( - node: &OperatorNode, - assignment: &MaterializationAssignment, - seen: &mut HashMap<*const OperatorNode, bool>, -) -> bool { - let key = node as *const OperatorNode; - if let Some(&cached) = seen.get(&key) { - return cached; - } - let upstream = |seen: &mut HashMap<*const OperatorNode, bool>| { - node.children() - .iter() - .any(|child| forces_query_time(child, assignment, seen)) - }; - let forced = match &node.operator { - _ if node.timing == Some(ExecutionTiming::QueryTime) => true, - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - | Operator::ASAP(ASAPOp::EvaluatePopulation { .. }) => true, - _ if node.timing.is_none() - && (is_materialization_point(&node.operator) || assignment.get(node).is_some()) => - { - let upstream = upstream(seen); - match assignment.resolve(node, upstream) { - Ok(Materialization::NotMaterialized) - if !is_materialization_point(&node.operator) => - { - upstream - } - Ok(m) => m.timing() == ExecutionTiming::QueryTime, - Err(_) => true, - } - } - Operator::NonASAP(NonASAPOp::BinaryOp { lhs, rhs, .. }) - if node - .schema - .fields - .iter() - .any(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) - && per_series_rows(lhs).is_none_or(|rows| per_series_rows(rhs) != Some(rows)) => - { - true - } - _ => upstream(seen), - }; - seen.insert(key, forced); - forced -} - -/// Write the timings of `assignment` into every node reachable from `root`, -/// top-down, then validate every edge. Returns the timed copy of `root`; -/// `cache` carries the sharing across the roots of one workload. -pub fn apply_materialization_timings( - root: &Rc, - assignment: &MaterializationAssignment, - cache: &mut NodeToTimedNodeCache, -) -> Result, ExecutionDataStateError> { - let mut forced = HashMap::new(); - let timed = write( - root, - ExecutionTiming::QueryTime, - assignment, - cache, - &mut forced, - )?; - // A root stored at ingestion time is read by the query; one that runs at - // ingestion time without being stored there produces rows nothing reads. - if timed.timing == Some(ExecutionTiming::IngestionTime) - && !matches!(timed.materialization, Materialization::MatAtIngestion(_)) - && data_state(&timed).map(|s| s.primitive) == Some(DataPrimitive::Raw) - { - return Err(ExecutionDataStateError::UnstoredIngestionRoot { - operator: timed.operator.kind_name(), - }); - } - validate(&timed, &mut HashMap::new())?; - Ok(timed) -} - -/// [`apply_materialization_timings`] over every root of one workload with -/// one cache, so a node several roots share is timed once and stays one -/// node. A standalone scalar root has each operator it reads timed, and is -/// rebuilt over the timed nodes. -pub fn apply_materialization_timings_to_roots( - roots: &[QueryRoot], - assignment: &MaterializationAssignment, -) -> Result, ExecutionDataStateError> { - let mut cache = NodeToTimedNodeCache::new(); - roots - .iter() - .map(|root| { - Ok(match root { - QueryRoot::Operator(node) => QueryRoot::Operator(apply_materialization_timings( - node, assignment, &mut cache, - )?), - QueryRoot::Scalar(expr) => { - for node in expr.operator_refs() { - apply_materialization_timings(node, assignment, &mut cache)?; - } - QueryRoot::Scalar(expr.map_operator_refs(&mut |node| { - Rc::clone(cache.timed(node).expect("scalar reference was timed")) - })) - } - }) - }) - .collect() -} - -/// Validate the sub-DAG below `root` with every sub-DAG `MatAtIngestion` -/// where the DAG allows it ([`MaterializationAssignment::with_default`]) and -/// `root` consumed at `root_timing`. For planning-time legality checks of a -/// candidate before it is assembled into a workload DAG: a candidate must stay -/// executable if materialization later puts its states at ingestion time. -/// Nothing is kept. -#[cfg(any(test, feature = "test-support"))] -pub fn validate_at_ingestion( - root: &Rc, - root_timing: ExecutionTiming, -) -> Result<(), ExecutionDataStateError> { - let assignment = all_at_ingestion(); - let mut cache = NodeToTimedNodeCache::new(); - let mut forced = HashMap::new(); - let timed = write(root, root_timing, &assignment, &mut cache, &mut forced)?; - validate(&timed, &mut HashMap::new()) -} - -/// The data state `node` produces with every sub-DAG `MatAtIngestion` where -/// the DAG allows it, when its consumer runs at `consumer` — the planning-time answer to -/// "what does this candidate's output look like", consistent with -/// [`validate_at_ingestion`]. -#[cfg(any(test, feature = "test-support"))] -pub fn planned_data_state( - node: &Rc, - consumer: ExecutionTiming, -) -> ExecutionDataState { - let mut forced = HashMap::new(); - let (timing, _) = placement(node, consumer, &all_at_ingestion(), &mut forced) - .expect("a default assignment never fails"); - ExecutionDataState { - timing, - primitive: primitive(node), - } -} - -/// The assignment of [`validate_at_ingestion`] and [`planned_data_state`]. -#[cfg(any(test, feature = "test-support"))] -fn all_at_ingestion() -> MaterializationAssignment { - MaterializationAssignment::with_default(Materialization::MatAtIngestion(Medium::Memory)) -} - -/// The timing and materialization `node` takes when its consumer runs at -/// `consumer`. -fn placement( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - forced: &mut HashMap<*const OperatorNode, bool>, -) -> Result<(ExecutionTiming, Materialization), ExecutionDataStateError> { - // A placement fixed when the candidate was built (an exact-state read - // boundary that must run at query time, or one that feeds maintenance) - // is honored; a conflicting consumer is rejected by validation. - let upstream_at_query_time = |forced: &mut HashMap<*const OperatorNode, bool>| { - node.children() - .iter() - .any(|child| forces_query_time(child, assignment, forced)) - }; - if let Some(placed) = node.timing { - if placed == ExecutionTiming::IngestionTime && is_filtered_build(node) { - return Err(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: node.operator.kind_name(), - }); - } - if placed == ExecutionTiming::IngestionTime && upstream_at_query_time(forced) { - return Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }); - } - // An explicit assignment is never silently dropped: one that agrees - // with the fixed timing is kept, one that needs the other timing is - // an error. - return match assignment.get(node) { - Some(m) if m != Materialization::NotMaterialized && m.timing() != placed => Err( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: node.operator.kind_name(), - fixed: placed, - materialization: m, - }, - ), - Some(m) => Ok((placed, m)), - None => Ok((placed, node.materialization)), - }; - } - Ok(match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - | Operator::ASAP(ASAPOp::EvaluatePopulation { .. }) => match assignment.get(node) { - Some(m) if m.timing() == ExecutionTiming::IngestionTime => { - return Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }) - } - Some(m) => (ExecutionTiming::QueryTime, m), - None => (ExecutionTiming::QueryTime, Materialization::NotMaterialized), - }, - _ if is_materialization_point(&node.operator) || assignment.get(node).is_some() => { - let upstream = upstream_at_query_time(forced); - match assignment.resolve(node, upstream)? { - Materialization::NotMaterialized if !is_materialization_point(&node.operator) => { - (consumer, Materialization::NotMaterialized) - } - materialization => (materialization.timing(), materialization), - } - } - _ => (consumer, Materialization::NotMaterialized), - }) -} - -/// The timing `node` takes when its consumer runs at `consumer`, by the -/// placement rules [`apply_materialization_timings`] applies, for Stage 2's -/// `split_shared_by_timing`: an assignment that cannot be honored is -/// reported later by [`apply_materialization_timings`]. -/// `query_time_forced` caches, per node, whether something at or below it -/// can only run at query time: pass an empty map and reuse it across the -/// calls of one assignment. -pub fn timing_under_consumer( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - query_time_forced: &mut HashMap<*const OperatorNode, bool>, -) -> ExecutionTiming { - placement(node, consumer, assignment, query_time_forced) - .map_or(ExecutionTiming::QueryTime, |(timing, _)| timing) -} - -fn write( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - cache: &mut NodeToTimedNodeCache, - forced: &mut HashMap<*const OperatorNode, bool>, -) -> Result, ExecutionDataStateError> { - let (timing, materialization) = placement(node, consumer, assignment, forced)?; - if let Some(done) = cache.timed_node_of.get(&Rc::as_ptr(node)) { - let previous = done.timing.expect("memoized node is timed"); - if previous != timing { - return Err(ExecutionDataStateError::ConflictingTiming { - first: ExecutionDataState { - timing: previous, - primitive: data_state(done).map_or(DataPrimitive::Raw, |s| s.primitive), - }, - second: ExecutionDataState { - timing, - primitive: data_state(done).map_or(DataPrimitive::Raw, |s| s.primitive), - }, - }); - } - return Ok(Rc::clone(done)); - } - let mut error = None; - let operator = - node.operator.map_children( - |child| match write(child, timing, assignment, cache, forced) { - Ok(timed) => timed, - Err(e) => { - error.get_or_insert(e); - Rc::clone(child) - } - }, - ); - if let Some(e) = error { - return Err(e); - } - let timed = Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(Some(timing)) - .with_materialization(materialization), - ); - cache - .timed_node_of - .insert(Rc::as_ptr(node), Rc::clone(&timed)); - Ok(timed) -} - -fn state_of(node: &OperatorNode) -> ExecutionDataState { - data_state(node).expect("timed node") -} - -/// Check every edge below `node` against the module-level rules. -fn validate( - node: &Rc, - seen: &mut HashMap<*const OperatorNode, ()>, -) -> Result<(), ExecutionDataStateError> { - if seen.insert(Rc::as_ptr(node), ()).is_some() { - return Ok(()); - } - let timing = node.timing.expect("timed node"); - match &node.operator { - Operator::ASAP(op) => validate_asap(node, op, timing)?, - Operator::NonASAP(op) => validate_non_asap(node, op, timing)?, - } - for child in node.children() { - validate(child, seen)?; - } - Ok(()) -} - -fn is_exact_accumulator_state(schema: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &schema.fields { - match &field.dtype { - FieldDataType::Plain(_) | FieldDataType::ExactAggregate(..) => {} - other => { - return Err(ExecutionDataStateError::UnsupportedStateComposition { - family: format!("{other:?}"), - }) - } - } - } - Ok(()) -} - -fn validate_asap( - node: &OperatorNode, - op: &ASAPOp, - timing: ExecutionTiming, -) -> Result<(), ExecutionDataStateError> { - match op { - ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. } => { - let edge = match op { - ASAPOp::SummaryTumblingWindowChain { .. } => "SummaryTumblingWindowChain.child", - _ => "SummaryAgg.child", - }; - let avail = state_of(child); - match avail { - ExecutionDataState::INGESTION_ROWS | ExecutionDataState::QUERY_ROWS => {} - s if s.primitive == DataPrimitive::SummaryState => { - is_exact_accumulator_state(&child.schema)? - } - other => { - return Err( - ExecutionDataStateError::QueryTimeValueUnderIngestionSummary { - edge, - child: other, - }, - ) - } - } - // A query-time build reads an ingestion-time input (rows or - // exact-accumulator state) only from where it is stored, as a - // `NonASAP` consumer does. - if timing == ExecutionTiming::QueryTime - && avail.timing == ExecutionTiming::IngestionTime - && child.materialization.timing() != ExecutionTiming::IngestionTime - { - return Err( - ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { - edge, - operator: child.operator.kind_name(), - }, - ); - } - if timing == ExecutionTiming::IngestionTime - && avail.timing == ExecutionTiming::QueryTime - { - return Err( - ExecutionDataStateError::QueryTimeValueUnderIngestionSummary { - edge, - child: avail, - }, - ); - } - Ok(()) - } - ASAPOp::SummaryEstimate { summary_input, .. } => { - let s = state_of(summary_input); - if s.primitive != DataPrimitive::SummaryState { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "SummaryEstimate.summary_input", - child: s, - }); - } - if timing != ExecutionTiming::QueryTime { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "SummaryEstimate", - child: state_of(node), - }); - } - Ok(()) - } - ASAPOp::FinalizeExactAccumulator { child } => { - let s = state_of(child); - if s.primitive != DataPrimitive::SummaryState - || is_exact_accumulator_state(&child.schema).is_err() - || (timing == ExecutionTiming::IngestionTime && s.timing != timing) - { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "FinalizeExactAccumulator.child", - child: s, - }); - } - Ok(()) - } - ASAPOp::MaintainPopulation { child, population } => { - let valid = population.matches_node(child) - && state_of(child) - == ExecutionDataState { - timing, - primitive: DataPrimitive::Raw, - }; - if !valid { - return Err(ExecutionDataStateError::InvalidMaintainedPopulation); - } - Ok(()) - } - ASAPOp::EvaluatePopulation { child, evaluation } => { - let valid = timing == ExecutionTiming::QueryTime - && matches!( - &child.operator, - Operator::ASAP(ASAPOp::MaintainPopulation { population, .. }) - if population.supports(evaluation) - && child.timing.is_some() - ); - if !valid { - return Err(ExecutionDataStateError::InvalidMaintainedPopulation); - } - Ok(()) - } - ASAPOp::SummaryMerge { children, .. } => { - for child in children { - let s = state_of(child); - if s.primitive != DataPrimitive::SummaryState - || (timing == ExecutionTiming::IngestionTime && s.timing != timing) - { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "SummaryMerge.children", - child: s, - }); - } - } - Ok(()) - } - ASAPOp::SummarySubtract { .. } - | ASAPOp::SummaryDelete { .. } - | ASAPOp::SummaryJoin { .. } - | ASAPOp::Extension { .. } => Err(ExecutionDataStateError::UnimplementedOperator { - operator: op.kind_name(), - }), - } -} - -fn check_plain_or_exact_values(input: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &input.fields { - if !matches!( - field.dtype, - FieldDataType::Plain(_) | FieldDataType::ExactAggregate(..) - ) { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -fn check_all_plain(input: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &input.fields { - if !field.is_plain() { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -/// Whether `node` is a filter that keeps whole summary states by their key -/// columns ([`selected_from`](crate::ir::operator::asap::selected_from)). -fn selects_states(node: &OperatorNode) -> bool { - crate::ir::operator::asap::selected_from(node).is_some() -} - -fn validate_non_asap( - node: &OperatorNode, - op: &NonASAPOp, - timing: ExecutionTiming, -) -> Result<(), ExecutionDataStateError> { - // Every input is rows at this node's own timing. Ingestion work never - // reads a query-time value; exact-accumulator state may pass through - // the projection-like operators unchanged. - for child in op.children() { - let s = state_of(child); - let passes_state = matches!( - op, - NonASAPOp::Project { .. } - | NonASAPOp::Filter { .. } - | NonASAPOp::Sort { .. } - | NonASAPOp::Limit { .. } - ) && s.primitive == DataPrimitive::SummaryState - && (is_exact_accumulator_state(&child.schema).is_ok() || selects_states(node)); - // A query-time consumer may read the stored output of a sub-DAG - // materialized at ingestion time. - let reads_stored = timing == ExecutionTiming::QueryTime - && child.materialization.timing() == ExecutionTiming::IngestionTime; - if (s.timing != timing && !reads_stored) - || (s.primitive != DataPrimitive::Raw && !passes_state) - { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: op.kind_name(), - child: s, - }); - } - } - match op { - // A key filter keeps whole summary states of any family. - NonASAPOp::Filter { .. } if selects_states(node) => {} - NonASAPOp::Project { child, .. } - | NonASAPOp::Filter { child, .. } - | NonASAPOp::Sort { child, .. } - | NonASAPOp::Limit { child, .. } => check_plain_or_exact_values(&child.schema)?, - NonASAPOp::Aggregate { - reduction, - measures, - child, - .. - } => { - let mut referenced: Vec = reduction - .group_keys() - .map(|keys| keys.keys().to_vec()) - .unwrap_or_default(); - for m in measures { - referenced.extend(m.input_cols()); - } - let implicit = measures.iter().any(|m| m.input_cols().is_empty()); - for (i, field) in child.schema.fields.iter().enumerate() { - if (implicit || referenced.contains(&i)) && !field.is_plain() { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - } - NonASAPOp::BinaryOp { - operator, lhs, rhs, .. - } => { - let is_div = matches!( - operator.kind, - BinaryOpKind::Arithmetic(crate::ir::scalar::ArithmeticOpKind::Div) - ); - if (operator.checked_relative_division && operator.checked_finite_division) - || ((operator.checked_relative_division || operator.checked_finite_division) - && (timing != ExecutionTiming::QueryTime || !is_div)) - { - return Err(ExecutionDataStateError::InvalidCheckedDivision); - } - if timing == ExecutionTiming::IngestionTime { - // The opaque series identity is a non-null Utf8 key; every - // other column is a non-null Float64 value or timestamp. - let plain_float_or_ts = |schema: &Schema| { - schema.fields.iter().all(|field| { - !field.nullable - && if field.name == crate::ir::schema::PROMQL_SERIES_IDENTITY { - field.dtype == FieldDataType::Plain(DataType::Utf8) - } else { - matches!( - field.dtype, - FieldDataType::Plain(DataType::Float64 | DataType::Timestamp) - ) - } - }) - }; - let float_count = node - .schema - .fields - .iter() - .filter(|f| matches!(f.dtype, FieldDataType::Plain(DataType::Float64))) - .count(); - if operator.vector_match.is_some() - || !matches!(operator.kind, BinaryOpKind::Arithmetic(_)) - || lhs.schema != rhs.schema - || lhs.schema != node.schema - || node - .schema - .fields - .iter() - .filter(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) - .count() - > 1 - || (node - .schema - .fields - .iter() - .any(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) - && per_series_rows(lhs) - .is_none_or(|rows| per_series_rows(rhs) != Some(rows))) - || !plain_float_or_ts(&node.schema) - || float_count != 1 - { - return Err(ExecutionDataStateError::InvalidIngestionTimeBinary); - } - } - } - _ => { - for child in op.children() { - check_all_plain(&child.schema)?; - } - } - } - Ok(()) -} - -/// Ingestion-time arithmetic needs the same per-series population on both sides. -fn per_series_rows(node: &OperatorNode) -> Option<&OperatorNode> { - use crate::ir::schema::ExactKind; - match &node.operator { - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match &child.operator { - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: FieldDataType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), - reduction: crate::ir::operator::Reduction::PerEntity, - filter: None, - .. - }) => Some(child), - _ => None, - }, - Operator::NonASAP(NonASAPOp::BinaryOp { lhs, rhs, .. }) => { - let rows = per_series_rows(lhs)?; - (per_series_rows(rhs) == Some(rows)).then_some(rows) - } - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ir::operator::agg_intent::AggIntent; - use crate::ir::operator::non_asap::NonASAPOp; - use crate::ir::operator::operator_properties::{Reduction, Source}; - use crate::ir::scalar::expr_ir::ColumnRef; - use crate::ir::schema::sketch::{ - ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, - }; - use crate::ir::schema::Field; - - fn scan_with(fields: Vec) -> Rc { - OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index(fields, 0, vec![]), - })) - .unwrap() - } - - fn scan() -> Rc { - scan_with(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("zone", DataType::Utf8, true), - ]) - } - - fn kll() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) - } - - fn exact_sum() -> FieldDataType { - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - } - - fn agg(child: Rc, family: FieldDataType) -> Rc { - std::rc::Rc::new( - OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: family.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }), - Schema::lifted(vec![Field::new("state", family, false)], None), - ) - .with_guarantee(None), - ) - } - - fn estimate(child: Rc) -> Rc { - std::rc::Rc::new( - OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: child, - query: SketchStatistic::Quantile { q: 0.99 }, - }), - Schema::lifted( - vec![Field::plain("quantile_0_99", DataType::Float64, false)], - None, - ), - ) - .with_guarantee(None), - ) - } - - fn aggregate(measure: AggIntent, child: Rc) -> Rc { - OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![measure], - output_names: vec![], - filters: vec![], - having: None, - child, - })) - .unwrap() - } - - fn max(child: Rc) -> Rc { - aggregate(AggIntent::Max { col: None }, child) - } - - /// `node` with its timing fixed in advance, as a candidate builder does - /// for an operator that must feed maintenance. - fn placed_at_ingestion(node: Rc) -> Rc { - Rc::new( - (*node) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)), - ) - } - - /// Apply with every sub-DAG `MatAtIngestion` where the DAG allows it, the - /// placement whose edge rules these tests exercise. - fn apply(root: &Rc) -> Result, ExecutionDataStateError> { - apply_materialization_timings(root, &all_at_ingestion(), &mut NodeToTimedNodeCache::new()) - } - - const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - - fn child(node: &Rc) -> Rc { - Rc::clone(node.children()[0]) - } - - /// Without a materialization decision, a summary is `NotMaterialized` and - /// it and its input run at query time; an explicit choice overrides the - /// default and is recorded on the timed node. - #[test] - fn default_assignment_materializes_nothing() { - let summary = agg(scan(), kll()); - let root = apply_materialization_timings( - &summary, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!( - data_state(&root), - Some(ExecutionDataState { - timing: ExecutionTiming::QueryTime, - primitive: DataPrimitive::SummaryState, - }) - ); - assert_eq!(root.materialization, Materialization::NotMaterialized); - assert_eq!( - data_state(&child(&root)), - Some(ExecutionDataState::QUERY_ROWS) - ); - for (materialization, timing) in [ - (AT_INGESTION, ExecutionTiming::IngestionTime), - ( - Materialization::MatAtQuery(Medium::Disk), - ExecutionTiming::QueryTime, - ), - ] { - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, materialization); - let root = apply_materialization_timings( - &summary, - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!(root.timing, Some(timing)); - assert_eq!(root.materialization, materialization); - assert_eq!(child(&root).timing, Some(timing)); - assert_eq!( - child(&root).materialization, - Materialization::NotMaterialized - ); - } - } - - /// #509 Algorithm 3, line 13: an explicit `MatAtIngestion` over a state - /// that only exists at query time is an error, not a silent move to - /// query time. - #[test] - fn ingestion_time_node_over_query_time_upstream_is_rejected() { - let inner = agg(scan(), exact_sum()); - let outer = agg(Rc::clone(&inner), kll()); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&outer, AT_INGESTION); - assert_eq!( - apply_materialization_timings( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "SummaryAgg" - }) - ); - assignment.set(&inner, AT_INGESTION); - let outer = agg(Rc::clone(&inner), kll()); - assignment.set(&outer, AT_INGESTION); - let root = apply_materialization_timings( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!(child(&root).materialization, AT_INGESTION); - assert_eq!(child(&child(&root)).materialization, AT_INGESTION); - } - - #[test] - fn summary_agg_input_runs_at_ingestion_time() { - let root = apply(&agg(scan(), kll())).unwrap(); - assert_eq!( - data_state(&root), - Some(ExecutionDataState::INGESTION_SUMMARY) - ); - assert_eq!( - data_state(&child(&root)), - Some(ExecutionDataState::INGESTION_ROWS) - ); - } - - #[test] - fn exact_accumulator_state_may_feed_another_summary_agg() { - let inner = agg(scan(), exact_sum()); - assert!(apply(&estimate(agg(inner, kll()))).is_ok()); - } - - /// A state not at ingestion time leaves the summary that reads it, when - /// that summary has no explicit assignment, `NotMaterialized` at query - /// time, as an evaluation below it would. - #[test] - fn query_time_state_moves_its_consuming_summary_to_query_time() { - let inner = agg(scan(), exact_sum()); - let outer = agg(Rc::clone(&inner), kll()); - let mut assignment = all_at_ingestion(); - assignment.set(&inner, Materialization::NotMaterialized); - let root = apply_materialization_timings( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let outer = child(&root); - assert_eq!(outer.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(outer.materialization, Materialization::NotMaterialized); - assert_eq!(child(&outer).timing, Some(ExecutionTiming::QueryTime)); - } - - #[test] - fn evaluation_can_feed_summary_construction_at_query_time() { - let inner = estimate(agg(scan(), kll())); - let root = apply(&estimate(agg(inner, kll()))).unwrap(); - assert_eq!(child(&root).timing, Some(ExecutionTiming::QueryTime)); - } - - /// Any non-ASAP operator over an evaluation runs at query time. - #[test] - fn query_time_operation_over_evaluation_is_legal_and_root_is_evaluation() { - let evaluation = || estimate(agg(scan(), kll())); - let sorted = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Sort { - keys: vec![], - partition_by: Default::default(), - child: evaluation(), - })) - .unwrap(); - for root in [max(evaluation()), sorted] { - let root = apply(&root).unwrap(); - assert_eq!(data_state(&root), Some(ExecutionDataState::QUERY_ROWS)); - } - } - - #[test] - fn query_time_values_can_feed_query_time_summary_construction() { - let post = max(estimate(agg(scan(), kll()))); - let root = apply(&estimate(agg(post, kll()))).unwrap(); - assert_eq!(child(&root).timing, Some(ExecutionTiming::QueryTime)); - } - - #[test] - fn function_under_summary_agg_is_legal_but_not_at_root() { - let operation = placed_at_ingestion(max(scan())); - assert_eq!( - apply(&operation).err(), - Some(ExecutionDataStateError::UnstoredIngestionRoot { - operator: "Aggregate" - }) - ); - let root = apply(&estimate(agg(operation, kll()))).unwrap(); - let timed_operation = child(&child(&root)); - assert_eq!( - data_state(&timed_operation), - Some(ExecutionDataState::INGESTION_ROWS) - ); - } - - #[test] - fn function_over_evaluation_is_rejected() { - let operation = placed_at_ingestion(max(estimate(agg(scan(), kll())))); - assert_eq!( - apply(&estimate(agg(operation, kll()))).err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "Aggregate" - }) - ); - } - - /// One shared sub-DAG reached as ingestion-time input and as query-time - /// input cannot be executed once for both (Stage 2's - /// `split_shared_by_timing` copies it first). - #[test] - fn a_shared_subtree_reached_at_two_timings_conflicts() { - let shared = scan(); - let root = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![ - max(estimate(agg(Rc::clone(&shared), kll()))), - max(Rc::clone(&shared)), - ], - })) - .unwrap(); - assert_eq!( - apply(&root).err(), - Some(ExecutionDataStateError::ConflictingTiming { - first: ExecutionDataState::INGESTION_ROWS, - second: ExecutionDataState::QUERY_ROWS, - }) - ); - } - - /// A finalize pinned to ingestion time (as fixed-window rate candidates - /// build it) over a state that is not `MatAtIngestion` names the missing - /// upstream, not an illegal edge. - #[test] - fn ingestion_pinned_finalize_over_query_time_state_is_upstream_not_at_ingestion() { - let finalize = placed_at_ingestion( - OperatorNode::new_shared(crate::ir::Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { - child: agg(scan(), exact_sum()), - }, - )) - .unwrap(), - ); - assert_eq!( - apply_materialization_timings( - &estimate(agg(finalize, kll())), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "FinalizeExactAccumulator" - }) - ); - } - - /// Any node may be materialized, not only a summary state: stored rows - /// at ingestion time are read by a query-time consumer, and a node that - /// can only run at query time cannot be stored at ingestion time. - #[test] - fn any_node_may_be_materialized() { - let tumbling_windows = max(scan()); - let root = max(Rc::clone(&tumbling_windows)); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&tumbling_windows, AT_INGESTION); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).materialization, AT_INGESTION); - assert_eq!(child(&timed).timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!( - child(&child(&timed)).timing, - Some(ExecutionTiming::IngestionTime) - ); - - let over_evaluation = max(estimate(agg(scan(), kll()))); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&over_evaluation, AT_INGESTION); - assert_eq!( - apply_materialization_timings( - &over_evaluation, - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "Aggregate" - }) - ); - } - - /// A summary build with a filter never runs at ingestion time: an - /// ingestion-time build reads raw data samples, which carry none of the - /// query's columns the filter reads. An explicit `MatAtIngestion` is an - /// error; the all-at-ingestion default leaves it at query time. - #[test] - fn a_filtered_summary_build_never_runs_at_ingestion_time() { - use crate::ir::scalar::expr_ir::ScalarValue; - use crate::ir::{Predicate, ScalarExpr}; - let plain = agg(scan(), kll()); - let Operator::ASAP(ASAPOp::SummaryAgg { - child: rows, - family, - input, - reduction, - grouping, - .. - }) = plain.operator.clone() - else { - unreachable!() - }; - let filtered = Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child: rows, - family, - input, - reduction, - grouping, - filter: Some(Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true)))), - }), - plain.schema.clone(), - ) - .with_guarantee(None), - ); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&filtered, AT_INGESTION); - assert_eq!( - apply_materialization_timings( - &estimate(Rc::clone(&filtered)), - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: "SummaryAgg" - }) - ); - let timed = apply(&estimate(filtered)).unwrap(); - assert_eq!(child(&timed).timing, Some(ExecutionTiming::QueryTime)); - assert_eq!( - child(&timed).materialization, - Materialization::NotMaterialized - ); - assert!(validate_at_ingestion(&child(&timed), ExecutionTiming::QueryTime).is_ok()); - } - - fn finalize(child: Rc) -> Rc { - OperatorNode::new_shared(crate::ir::Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child }, - )) - .unwrap() - } - - /// The materialization the exported DAG records for `node`'s operator kind. - fn exported(root: &Rc, kind: &str) -> Materialization { - let dag = crate::ir::physical_export::compile_physical_asap_dag(root).unwrap(); - dag.validate().unwrap(); - dag.nodes - .iter() - .find(|n| n.payload.kind_name() == kind) - .unwrap() - .materialization - } - - /// A node whose timing its candidate fixed (the query-time read boundary - /// of `finalize_query_candidate`) keeps an explicit assignment that agrees - /// with that timing and rejects one that needs the other timing. - #[test] - fn fixed_timing_keeps_an_agreeing_assignment_and_rejects_a_conflict() { - let state = agg(scan(), exact_sum()); - let read = Rc::new( - (*finalize(Rc::clone(&state))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - let stored = Materialization::MatAtQuery(Medium::Disk); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&read, stored); - let timed = - apply_materialization_timings(&read, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(timed.materialization, stored); - assert_eq!(exported(&timed, "FinalizeExactAccumulator"), stored); - - let mut assignment = MaterializationAssignment::default(); - assignment.set(&state, AT_INGESTION); - assignment.set(&read, AT_INGESTION); - assert_eq!( - apply_materialization_timings(&read, &assignment, &mut NodeToTimedNodeCache::new()) - .err(), - Some( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: "FinalizeExactAccumulator", - fixed: ExecutionTiming::QueryTime, - materialization: AT_INGESTION, - } - ) - ); - } - - /// The Debug output names each assigned node by its operator kind, not - /// its whole sub-DAG. - #[test] - fn debug_names_each_assigned_node_by_its_kind() { - let mut assignment = MaterializationAssignment::default(); - assignment.set(&agg(scan(), kll()), AT_INGESTION); - let text = format!("{assignment:?}"); - assert!(text.contains("SummaryAgg@"), "{text}"); - assert!(!text.contains("fields"), "{text}"); - } - - /// A node fixed at ingestion time by its candidate rejects a `MatAtQuery` - /// assignment, which needs the other timing. - #[test] - fn ingestion_fixed_node_rejects_mat_at_query() { - let fixed = placed_at_ingestion(max(scan())); - let root = max(Rc::clone(&fixed)); - let at_query = Materialization::MatAtQuery(Medium::Memory); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&fixed, at_query); - assert_eq!( - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .err(), - Some( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: "Aggregate", - fixed: ExecutionTiming::IngestionTime, - materialization: at_query, - } - ) - ); - } - - /// An explicit `NotMaterialized` on a node whose timing its candidate - /// fixed agrees with either timing: the node keeps its fixed timing and - /// records `NotMaterialized`. - #[test] - fn explicit_not_materialized_keeps_a_fixed_timing() { - let read = Rc::new( - (*finalize(agg(scan(), exact_sum()))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - let root = max(Rc::clone(&read)); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&read, Materialization::NotMaterialized); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let read = child(&timed); - assert_eq!(read.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(read.materialization, Materialization::NotMaterialized); - } - - /// A query-time summary build cannot read ingestion-time rows that are - /// not stored: at query time they no longer exist. Storing the rows node - /// at ingestion time makes the same build valid (#738). - #[test] - fn query_time_summary_over_unstored_ingestion_rows_is_rejected() { - let rows = placed_at_ingestion(max(scan())); - let summary = agg(Rc::clone(&rows), kll()); - assert_eq!( - apply_materialization_timings( - &summary, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some( - ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { - edge: "SummaryAgg.child", - operator: "Aggregate", - } - ) - ); - - let mut assignment = MaterializationAssignment::default(); - assignment.set(&rows, AT_INGESTION); - let timed = - apply_materialization_timings(&summary, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).materialization, AT_INGESTION); - } - - /// The same holds for exact-accumulator state: a query-time build cannot - /// read an ingestion-time exact state that is not stored. - #[test] - fn query_time_summary_over_unstored_ingestion_state_is_rejected() { - let state = placed_at_ingestion(agg(scan(), exact_sum())); - let summary = agg(Rc::clone(&state), kll()); - assert_eq!( - apply_materialization_timings( - &summary, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some( - ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { - edge: "SummaryAgg.child", - operator: "SummaryAgg", - } - ) - ); - - let mut assignment = MaterializationAssignment::default(); - assignment.set(&state, AT_INGESTION); - let timed = - apply_materialization_timings(&summary, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).materialization, AT_INGESTION); - } - - /// A root stored at ingestion time is valid: the query reads the stored - /// output. A root that runs at ingestion time without being stored there - /// is not. - #[test] - fn a_root_may_be_stored_at_ingestion_time() { - let root = max(scan()); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&root, AT_INGESTION); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!(exported(&timed, "Aggregate"), AT_INGESTION); - - let state = agg(scan(), exact_sum()); - let root = finalize(Rc::clone(&state)); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&state, AT_INGESTION); - assignment.set(&root, AT_INGESTION); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!(exported(&timed, "FinalizeExactAccumulator"), AT_INGESTION); - - let mut assignment = MaterializationAssignment::default(); - let root = placed_at_ingestion(max(scan())); - assignment.set(&root, Materialization::NotMaterialized); - assert_eq!( - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .err(), - Some(ExecutionDataStateError::UnstoredIngestionRoot { - operator: "Aggregate" - }) - ); - } - - /// Both paired operands must be plain; an unrelated state column is not - /// an input. - #[test] - fn pearson_corr_checks_both_operand_states() { - let corr_over = |state_column: usize| { - let mut fields = vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("x", DataType::Float64, false), - Field::plain("y", DataType::Float64, false), - Field::plain("unused", DataType::Float64, false), - ]; - fields[state_column].dtype = kll(); - aggregate( - AggIntent::PearsonCorr { left: 1, right: 2 }, - scan_with(fields), - ) - }; - for operand in [1, 2] { - assert!(matches!( - validate_at_ingestion(&corr_over(operand), ExecutionTiming::QueryTime), - Err(ExecutionDataStateError::NonPlainOperand { .. }) - )); - } - validate_at_ingestion(&corr_over(3), ExecutionTiming::QueryTime).unwrap(); - } - // Typed derived updates retain one opaque series identity only when both - // operands cover the same series; arbitrary labels are never admitted. - #[test] - fn ingestion_time_binary_accepts_only_well_typed_series_identity() { - use crate::ir::operator::non_asap::BinaryOperator; - use crate::ir::operator::BinaryOpKind; - use crate::ir::scalar::ArithmeticOpKind; - let identity = crate::ir::schema::PROMQL_SERIES_IDENTITY; - // A finalized per-series Sum of `metric`'s rows. - let operand = |metric: &str, schema: &Schema| { - let rows = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let mut state = schema.clone(); - state.fields[0].dtype = exact_sum(); - let sum = Rc::new(OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: rows, - family: exact_sum(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - }), - state, - )); - Rc::new(OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: sum }), - schema.clone(), - )) - }; - let validate = |schema: Schema, rhs: &str| { - let binary = Rc::new(OperatorNode::with_schema( - crate::ir::Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - }, - return_bool: false, - lhs: operand("m", &schema), - rhs: operand(rhs, &schema), - }), - schema, - )); - validate_at_ingestion(&placed_at_ingestion(binary), ExecutionTiming::IngestionTime) - }; - let mut schema = Schema::lifted( - vec![ - Field::plain("value", DataType::Float64, false), - Field::plain("ts", DataType::Timestamp, false), - ], - None, - ); - schema.time_index = Some(1); - assert!(validate(schema.clone(), "m").is_ok()); - assert!( - validate(schema.clone(), "n").is_ok(), - "no identity to align" - ); - schema - .fields - .push(Field::plain(identity, DataType::Utf8, false)); - assert!(validate(schema.clone(), "m").is_ok()); - assert_eq!( - validate(schema.clone(), "n"), - Err(ExecutionDataStateError::InvalidIngestionTimeBinary), - "different selectors may cover different series" - ); - for mutation in 0..4 { - let mut invalid = schema.clone(); - match mutation { - 0 => invalid.fields[2].nullable = true, - 1 => invalid.fields[2].dtype = FieldDataType::Plain(DataType::Timestamp), - 2 => invalid.fields.push(invalid.fields[2].clone()), - _ => invalid.fields[2].name = "label".into(), - } - assert_eq!( - validate(invalid, "m"), - Err(ExecutionDataStateError::InvalidIngestionTimeBinary) - ); - } - } -} diff --git a/crates/types/src/ir/scalar/expr_ir.rs b/crates/types/src/ir/scalar/expr_ir.rs index 50606f185..3ee1dd56d 100644 --- a/crates/types/src/ir/scalar/expr_ir.rs +++ b/crates/types/src/ir/scalar/expr_ir.rs @@ -39,6 +39,9 @@ pub enum ScalarValue { Utf8(String), Boolean(bool), Null, + /// A point in time, in ms since the Unix epoch: PromQL `@ 1700000000` + /// is `Timestamp(1_700_000_000_000)`. + Timestamp(i64), /// A calendar duration, in Arrow's three independent fields. Not collapsed /// into one nanosecond count: a month is not a fixed number of nanoseconds, /// so `INTERVAL '1' MONTH` has no faithful scalar form. DataFusion's three diff --git a/crates/types/src/ir/scalar/mod.rs b/crates/types/src/ir/scalar/mod.rs index 42cada55a..d06c82d00 100644 --- a/crates/types/src/ir/scalar/mod.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -19,6 +19,7 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; +use crate::ir::operator::input::OperatorInput; use crate::ir::operator::node::OperatorNode; use crate::ir::scalar::scalar_type_rules::MapScalarFunction; use crate::ir::schema::{ColumnId, DataType, Schema}; @@ -353,7 +354,7 @@ impl ScalarExpr { } } -impl ScalarExpr { +impl ScalarExpr { /// Infer the `(DataType, nullable)` this expression produces against the /// input schema its owner evaluates it in. Unregistered functions are /// rejected. A reference to a field carrying summary state is an error: @@ -381,6 +382,7 @@ impl ScalarExpr { ScalarValue::Boolean(_) => (DataType::Bool, false), ScalarValue::Null => (DataType::Null, true), ScalarValue::Interval { .. } => (DataType::Interval, false), + ScalarValue::Timestamp(_) => (DataType::Timestamp, false), }, ScalarExpr::Compare { left, right, op, .. @@ -579,7 +581,7 @@ impl ScalarExpr { // `scalar(v)` is one float sample (NaN when the vector is not // exactly one series); a scalar subquery is its single column. ScalarExpr::PromqlScalarFromVector(node) => { - if node.result_kind != super::OperatorResultKind::InstantVector { + if node.result_kind() != super::OperatorResultKind::InstantVector { return Err(signature("scalar() requires an instant vector")); } (DataType::Float64, false) @@ -622,25 +624,25 @@ fn common_scalar_type(a: &DataType, b: &DataType) -> Result Result<(), SchemaDerivationError> { - if node.result_kind == super::OperatorResultKind::Relation { +fn relation(node: &C) -> Result<(), SchemaDerivationError> { + if node.result_kind() == super::OperatorResultKind::Relation { Ok(()) } else { Err(signature("SQL subquery requires a relation")) } } -fn scalar_subquery_field( - node: &OperatorNode, +fn scalar_subquery_field( + node: &C, ) -> Result<&crate::ir::schema::Field, SchemaDerivationError> { relation(node)?; - match node.schema.fields.as_slice() { + match node.schema().fields.as_slice() { [field] => Ok(field), _ => Err(signature("scalar subquery requires exactly one column")), } } -fn sql_function_type( +fn sql_function_type( name: &str, - args: &[ScalarExpr], + args: &[ScalarExpr], schema: &Schema, ) -> Result<(DataType, bool), SchemaDerivationError> { let types = args @@ -720,7 +722,10 @@ fn sql_function_type( /// Resolve the bounded canonical `asap_struct_field(struct, selector)` operation. /// Selectors are positive 1-based literal ordinals or exact literal field names. -pub fn struct_field_type(args: &[ScalarExpr], schema: &Schema) -> Result<(DataType, bool), String> { +pub fn struct_field_type( + args: &[ScalarExpr], + schema: &Schema, +) -> Result<(DataType, bool), String> { let [input, selector] = args else { return Err("struct field access requires a struct and constant selector".into()); }; @@ -757,8 +762,8 @@ pub fn struct_field_type(args: &[ScalarExpr], schema: &Schema) -> Result<(DataTy /// Resolve `asap_element_access(collection, index)`: Map access through the /// map function contract, List access with integer indices. -pub fn element_access_type( - args: &[ScalarExpr], +pub fn element_access_type( + args: &[ScalarExpr], schema: &Schema, ) -> Result<(DataType, bool), String> { let [input, index] = args else { @@ -792,7 +797,7 @@ pub fn element_access_type( #[cfg(test)] mod tests { use super::*; - use crate::ir::operator::operator_properties::Source; + use crate::ir::operator::arguments::Source; use crate::ir::schema::{Field, FieldDataType}; use crate::ir::{NonASAPOp, OperatorNode}; diff --git a/crates/types/src/ir/schema/aggregate_schema.rs b/crates/types/src/ir/schema/aggregate_schema.rs index 32b2a98ca..2c1113534 100644 --- a/crates/types/src/ir/schema/aggregate_schema.rs +++ b/crates/types/src/ir/schema/aggregate_schema.rs @@ -5,7 +5,7 @@ //! aggregate result; a PromQL per-series range reduction preserves labels //! and produces a Float64 sample value. This module derives schemas, not //! aggregate values or summary candidates. -use crate::ir::operator::operator_properties::*; +use crate::ir::operator::arguments::*; use crate::ir::operator::AggIntent; use crate::ir::scalar::ColumnRef; use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index 90796a76b..72e3e5627 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -15,24 +15,18 @@ pub mod aggregate_schema; pub mod error; -pub mod family_capability; -pub mod schema_support; -pub mod sketch; +pub mod series_identity; pub mod topk; pub use aggregate_schema::aggregate_output_schema; -pub use family_capability::{ - FamilyCapability, MergeInputRule, MergedError, StatisticKind, SupportedSummary, -}; -pub use sketch::{ - EntityIdentity, ExactKind, ExactParams, GroupingStrategy, HydraKind, HydraParams, - NonNegativeWeightProof, SamplingKind, SamplingParams, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, StatModelKind, StatModelParams, SummaryInputExpr, SummaryUpdate, - WaveletKind, WaveletParams, WeightDomain, EXACT_ACCUMULATOR_STATE_BYTES, -}; use serde::{Deserialize, Serialize}; +use crate::summary_type::{ + ExactKind, ExactParams, SamplingKind, SamplingParams, SketchAlgorithm, StatModelKind, + StatModelParams, WaveletKind, WaveletParams, +}; + /// Zero-based position of a column in a particular operator's input or output. /// /// During planning, the position indexes [`Schema::fields`] to obtain metadata; @@ -143,11 +137,10 @@ impl From> for Field { /// What a schema field carries: an ordinary readable value, or the summary / /// exact-accumulator state produced by a `SummaryAgg`. /// -/// Every non-`Plain` variant carries the physical state identity required by -/// that family (`Sketch` additionally carries its grouping layout), so the -/// type system can reject merges of incompatible summaries at plan -/// construction time — a `SummaryMerge` over `Sketch(Kll, …)` and -/// `Sketch(Cms, …)` inputs is a plan-time error, and a `Sketch(…)` can never +/// Every non-`Plain` variant is a summary type: its family (the variant) and +/// its algorithm, so the type system can reject merges of incompatible +/// summaries at plan construction time — a `SummaryMerge` over `Sketch(Kll)` +/// and `Sketch(Cms)` inputs is a plan-time error, and a `Sketch(…)` can never /// be confused for a `SummarySample(…)` even though both are "opaque summary /// state" at a glance. /// @@ -163,12 +156,12 @@ pub enum FieldDataType { /// Exact, mergeable accumulator state (`Sum`/`Count`/`Min`/`Max`/`Rate`/ /// `Increase`). Value consumers require an explicit finalization boundary. ExactAggregate(ExactKind, ExactParams), - /// Approximate sketch state (KLL/CMS/HLL/…), evaluated by a - /// `SummaryEstimate`. A [`SketchKind`] already carries the concrete - /// algorithm, params, and grouping layout committed to, not just its - /// category — a bound node needs to know it's specifically independent - /// KLL or shared Hydra-backed CMS, not merely "some sketch". - Sketch(SketchKind, GroupingStrategy), + /// Approximate sketch state (KLL/CMS/HLL/…, and the `Hydra*` algorithms, + /// one structure shared by every subpopulation), evaluated by a + /// `SummaryEstimate`. θ is not part of the type, so states of one + /// algorithm are one memo group whatever their size; θ is a candidate's + /// `LogicalProperties::parameters` (concepts doc, section 1). + Sketch(SketchAlgorithm), /// A summary sample: the subset of input rows a [`SamplingKind`] keeps, /// such as a reservoir. Not raw data samples. Serialized as `Sample`. #[serde(rename = "Sample")] diff --git a/crates/types/src/ir/schema/schema_support.rs b/crates/types/src/ir/schema/series_identity.rs similarity index 96% rename from crates/types/src/ir/schema/schema_support.rs rename to crates/types/src/ir/schema/series_identity.rs index 913ab8728..61bc53e69 100644 --- a/crates/types/src/ir/schema/schema_support.rs +++ b/crates/types/src/ir/schema/series_identity.rs @@ -64,7 +64,7 @@ pub fn with_promql_series_identity( // A series has at most one sample per timestamp, so the full // identity and the timestamp key each row. Pass 2's // identical-expression rule reads this key: only a keyed - // node is legal to share (Stage 0 hash-consing, `cse` in `asap_logical_asap_optimization`). + // node is legal to share (hash-consing, `Memo::find_or_add_group`). if let Some(time) = schema.time_index { schema.add_unique_key(vec![schema.fields.len() - 1, time]); } @@ -92,8 +92,6 @@ pub fn with_promql_series_identity( } let mut rebuilt = OperatorNode::new(operator).map_err(|e| e.to_string())?; rebuilt.guarantee = node.guarantee.clone(); - rebuilt.timing = node.timing; - rebuilt.materialization = node.materialization; let rebuilt = Rc::new(rebuilt); cache.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); Ok(rebuilt) diff --git a/crates/types/src/ir/schema/sketch.rs b/crates/types/src/ir/schema/sketch.rs deleted file mode 100644 index a2c3a21e1..000000000 --- a/crates/types/src/ir/schema/sketch.rs +++ /dev/null @@ -1,840 +0,0 @@ -use serde::{Deserialize, Serialize}; - -use crate::ir::scalar::ColumnRef; - -// ── Exact accumulators ────────────────────────────────────────────────────── - -/// Bytes the executor holds per group for an exact accumulator of any -/// [`ExactKind`]: one element of its column of fixed-size states (the -/// executor's `ScalarState`, whose largest variant, a counter's, is 48 -/// bytes). Plan selection prices exact state with it; an executor test -/// asserts it equals that state's size. -pub const EXACT_ACCUMULATOR_STATE_BYTES: u64 = 56; - -/// An exact, mergeable accumulator family — zero approximation error. The -/// partial state built for one of these *is* the answer; no -/// `SummaryEstimate` evaluation is needed to get a value out of it. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -pub enum ExactKind { - /// Exact sum accumulator (mergeable by addition). - Sum, - /// Exact count accumulator (mergeable by addition). - Count, - /// Exact minimum accumulator (mergeable by comparison). - Min, - /// Exact maximum accumulator (mergeable by comparison). - Max, - /// Exact increase accumulator (counter-reset-aware delta). - Increase, - /// Rate accumulator (increase / time window duration). - Rate, - /// Instant-rate accumulator retaining the final two timestamped samples. - IRate, -} - -/// Parameters for an [`ExactKind`] accumulator. All exact accumulators have -/// fixed semantics — no tuning parameters — so each variant carries none; -/// kept as a per-kind enum (mirroring [`SketchParams`]) so a mismatched -/// `(kind, params)` pair is still a type error, not a runtime check. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum ExactParams { - Sum, - Count, - Min, - Max, - Increase, - Rate, - IRate, -} - -// ── Approximate sketches ───────────────────────────────────────────────────── - -/// A specific sketch algorithm — bounded error, sized by its -/// [`SketchParams`]. It is `asap_sketchlib`'s family enum; the statistics -/// each answers are its row of the family-capability table -/// ([`FamilyCapability`](crate::ir::schema::FamilyCapability)). The planner -/// sizes and offers only the families [`SketchParams`] has a variant for. -pub use asap_sketchlib::SketchAlgorithm; - -/// The largest KLL `k` `asap_sketchlib` builds: it silently caps a larger -/// `k` at this (`MAX_CACHEABLE_K` in its `sketches::kll`). -pub const KLL_MAX_K: u32 = 26_602; - -/// Concrete, catalog-validated parameters for a specific [`SketchAlgorithm`] -/// instance. The variant must correspond to the associated `SketchAlgorithm`; -/// mismatches are caught at post-ASAP bind time, before any later, -/// deployment-specific stage ever sees the plan. -/// -/// Equality compares the sketch that is built: a KLL's `failure_probability` -/// and a UnivMon's `distinct_failure_probability` record what it was sized -/// for, not its structure, so two KLLs with the same `k` are equal (they -/// share, and match the same offline evidence), as are two UnivMons of the -/// same shape. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum SketchParams { - UnivMon { - heap_size: u32, - sketch_rows: u32, - sketch_cols: u32, - layers: u8, - /// The failure probability the heap was sized for when this - /// UnivMon answers a distinct count; its relative error follows from - /// it and the heap. `None`: not sized for a distinct count, which - /// then has no guarantee. Ignored by equality. - #[serde(default, skip_serializing_if = "Option::is_none")] - distinct_failure_probability: Option, - }, - Kll { - k: u32, - /// The failure probability `k` was sized for, when below the - /// empirical 99% calibration's 0.01; `None` is that calibration. - /// Sizing never records `Some(δ)` for δ ≥ 0.01; a hand-built one is - /// read as a claim at that δ with the calibration's error bound, and - /// one outside (0, 1) gives no guarantee. Ignored by equality. - #[serde(default, skip_serializing_if = "Option::is_none")] - failure_probability: Option, - }, - Cms { - width: u32, - depth: u32, - }, - Hll { - precision: u8, - /// The relative error `precision` was sized for (#694): the error - /// bound of its guarantee, at which its failure probability is - /// evaluated. `None`: not sized for a target; its error bound is - /// the relative standard error `1.04/√m`. Ignored by equality. - #[serde(default, skip_serializing_if = "Option::is_none")] - relative_error: Option, - }, - DDSketch { - alpha: f64, - }, - CmsWithHeap { - width: u32, - depth: u32, - heap_size: u32, - }, - Kmv { - k: u32, - /// The relative error `k` was sized for (#694): the error bound of - /// its guarantee, at which its failure probability is evaluated. - /// `None`: not sized for a target; its error bound is the relative - /// standard error `1/√(k-2)`. Ignored by equality. - #[serde(default, skip_serializing_if = "Option::is_none")] - relative_error: Option, - }, - CountSketch { - width: u32, - depth: u32, - }, - CountSketchWithHeap { - width: u32, - depth: u32, - heap_size: u32, - }, -} - -impl PartialEq for SketchParams { - fn eq(&self, other: &Self) -> bool { - use SketchParams as P; - match (self, other) { - (P::Kll { k: a, .. }, P::Kll { k: b, .. }) => a == b, - ( - P::UnivMon { - heap_size: a0, - sketch_rows: a1, - sketch_cols: a2, - layers: a3, - .. - }, - P::UnivMon { - heap_size: b0, - sketch_rows: b1, - sketch_cols: b2, - layers: b3, - .. - }, - ) => (a0, a1, a2, a3) == (b0, b1, b2, b3), - ( - P::Cms { - width: a0, - depth: a1, - }, - P::Cms { - width: b0, - depth: b1, - }, - ) - | ( - P::CountSketch { - width: a0, - depth: a1, - }, - P::CountSketch { - width: b0, - depth: b1, - }, - ) => (a0, a1) == (b0, b1), - ( - P::CmsWithHeap { - width: a0, - depth: a1, - heap_size: a2, - }, - P::CmsWithHeap { - width: b0, - depth: b1, - heap_size: b2, - }, - ) - | ( - P::CountSketchWithHeap { - width: a0, - depth: a1, - heap_size: a2, - }, - P::CountSketchWithHeap { - width: b0, - depth: b1, - heap_size: b2, - }, - ) => (a0, a1, a2) == (b0, b1, b2), - (P::Hll { precision: a, .. }, P::Hll { precision: b, .. }) => a == b, - (P::DDSketch { alpha: a }, P::DDSketch { alpha: b }) => a == b, - (P::Kmv { k: a, .. }, P::Kmv { k: b, .. }) => a == b, - // Different variants. Listed by `self`'s variant, not `_`, so a - // new variant fails to compile here until it is compared above. - (P::UnivMon { .. }, _) - | (P::Kll { .. }, _) - | (P::Cms { .. }, _) - | (P::Hll { .. }, _) - | (P::DDSketch { .. }, _) - | (P::CmsWithHeap { .. }, _) - | (P::Kmv { .. }, _) - | (P::CountSketch { .. }, _) - | (P::CountSketchWithHeap { .. }, _) => false, - } - } -} - -/// A committed sketch choice: the concrete [`SketchAlgorithm`] and the -/// [`SketchParams`] realizing it. Sits between -/// [`FieldDataType::Sketch`](crate::ir::schema::FieldDataType::Sketch) -/// (the `Sketch` family as a whole, sibling to `SummarySample`/`Wavelet`/ -/// `StatModel`) and the bare algorithm. Which statistics a state of the -/// kind answers is the family-capability table's -/// ([`FamilyCapability`](crate::ir::schema::FamilyCapability)). -/// -/// Construct through [`SketchKind::new`], which rejects parameters of a -/// different algorithm. See `asap_logical_asap_optimization::summary_candidates` for the -/// `AggIntent -> [SketchAlgorithm]` candidate list. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct SketchKind { - algorithm: SketchAlgorithm, - params: SketchParams, -} - -impl SketchKind { - /// The kind of `algorithm` with `params`. Panics when `params` belongs - /// to a different algorithm; invalid committed sketch states cannot be - /// constructed. - pub fn new(algorithm: SketchAlgorithm, params: SketchParams) -> Self { - let valid_params = matches!( - (&algorithm, ¶ms), - (SketchAlgorithm::Kll, SketchParams::Kll { .. }) - | (SketchAlgorithm::UnivMon, SketchParams::UnivMon { .. }) - | (SketchAlgorithm::DDSketch, SketchParams::DDSketch { .. }) - | (SketchAlgorithm::Hll, SketchParams::Hll { .. }) - | (SketchAlgorithm::Kmv, SketchParams::Kmv { .. }) - | (SketchAlgorithm::Cms, SketchParams::Cms { .. }) - | ( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { .. } - ) - | ( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { .. } - ) - | ( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { .. } - ) - ); - assert!( - valid_params, - "SketchKind parameter mismatch: algorithm={algorithm:?}, params={params:?}" - ); - Self { algorithm, params } - } - - /// The algorithm this kind committed to. - pub fn algorithm(&self) -> &SketchAlgorithm { - &self.algorithm - } - - /// The parameters this kind committed to. - pub fn params(&self) -> &SketchParams { - &self.params - } -} - -// ── Sampling summaries ─────────────────────────────────────────────────────── - -/// A sampling summary family — retains an actual (weighted) subset of input -/// rows rather than a compressed sketch. The retained rows are its summary -/// sample ([`FieldDataType::SummarySample`](crate::ir::schema::FieldDataType::SummarySample)), -/// not the raw data samples a metric arrives as. Minimal -/// starting vocabulary: one family, extend as a deployment needs another -/// (stratified, weighted, …). -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -pub enum SamplingKind { - /// Reservoir sampling (mergeable via weighted reservoir merge; - /// uniform-random retained subset of a fixed size). - Reservoir, -} - -/// Parameters for a [`SamplingKind`] instance. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum SamplingParams { - Reservoir { - /// Reservoir capacity — the number of retained rows. - size: u32, - }, -} - -// ── Wavelet summaries ──────────────────────────────────────────────────────── - -/// A wavelet-transform-based summary family — a compressed coefficient -/// vector supporting approximate range-sum / histogram queries. Minimal -/// starting vocabulary: one basis, extend as a deployment needs another. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -pub enum WaveletKind { - /// Haar wavelet synopsis — the simplest, most common streaming basis. - Haar, -} - -/// Parameters for a [`WaveletKind`] instance. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum WaveletParams { - Haar { - /// Number of retained coefficients — the compression / accuracy knob. - coefficients: u32, - }, -} - -// ── Statistical-model summaries ────────────────────────────────────────────── - -/// A fitted statistical/parametric-model summary family — e.g. a distribution -/// fit used to answer approximate quantile/density queries without -/// retaining raw data samples. Deliberately open-ended: `family` names the model -/// family and is interpreted by whatever deployment builds/reads it, the -/// same "core doesn't enumerate every deployment shape" stance -/// `AggIntent::Extension` already takes for pre-ASAP intents. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -pub enum StatModelKind { - /// A parametric model fit to the data (e.g. a fitted distribution or - /// regression). `family` (in [`StatModelParams::Parametric`]) names - /// which one. - Parametric, -} - -/// Parameters for a [`StatModelKind`] instance. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum StatModelParams { - Parametric { - /// Deployment-interpreted model family name (e.g. - /// `"gaussian_mixture"`). Core has no fixed catalog of families — - /// see [`StatModelKind::Parametric`]. - family: String, - }, -} - -// ── Grouping strategy: per-subpopulation vs. shared multi-subpopulation -// (Hydra) — issue #256 ────────────────────────────────────────────────── -// -// A grouped aggregate (`GROUP BY city, quantile(...)`) has always implicitly -// built one independent summary instance per distinct `by` key — there was -// no type anywhere expressing that as a *choice* rather than a foregone -// conclusion. Hydra (Manousis et al., VLDB 2022 — see `HydraKind`'s own doc -// for the full citation and which variants are its actual proven -// construction) is the alternative: one shared structure serving every -// subpopulation instead of N independent instances, trading memory/build -// cost against per-subpopulation isolation. -// -// This axis is deliberately modeled here, alongside `SketchKind`/ -// `SamplingKind`/`WaveletKind`/`StatModelKind`, rather than as a new -// `SketchKind` (or `SamplingKind`, etc.) entry: it is orthogonal to *which* -// family/kind answers an intent — any family could in principle grow its own -// per-subpopulation vs. shared-multi-subpopulation variant, so it is a -// second, independent axis, not a member of any one family's own kind -// vocabulary. It is legal only for a grouped aggregate (a non-empty `by`) -// whose family has a Hydra variant (CMS or Count-Sketch); the planner does -// not offer it as a candidate yet (#703). - -/// A shared-multi-subpopulation summary family — one physical structure -/// serving every subpopulation of a grouped aggregate instead of one -/// independent instance per distinct `by` key. Named after Hydra (Manousis, -/// Cheng, Ben Basat, Liu, Sekar. "Enabling Efficient and General -/// Subpopulation Analytics in Multidimensional Data Streams." VLDB 2022). -/// -/// Orthogonal to [`SketchKind`]/[`SamplingKind`]/[`WaveletKind`]/ -/// [`StatModelKind`] the same way [`GroupingStrategy`] as a whole is -/// orthogonal to them. -/// -/// ## Which variants are the paper's own proven construction, and which aren't -/// -/// Hydra's accuracy proof (paper §4.5, Theorem 2) is over one specific -/// construction: hash each subpopulation into one of a shared w×r grid of -/// **linear, mergeable frequency-vector sketches** — the paper's own -/// heavy-hitter substrate is Count-Sketch (§4.3); Count-Min Sketch is the -/// same collision algebra — and bound the noise a colliding subpopulation's -/// estimate picks up from the others sharing its cell. -/// [`HydraKind::HydraCms`]/[`HydraKind::HydraCountSketch`] are exactly that -/// construction over this crate's existing `SketchAlgorithm::Cms`/ -/// `CountSketch`, so the paper's bound applies to them directly. -/// -/// [`HydraKind::HydraKll`] is **not** an instance of that proven -/// construction: KLL is an order-statistics sketch, not a linear frequency -/// vector, and has no analogous "sum the colliding contributions, bound the -/// noise" algebra. The paper is explicit that its own construction cannot -/// serve quantiles at all (§4.3: "A statistic that cannot directly be -/// estimated by Hydra-sketch is quantiles."). `HydraKll` remains available -/// as an explicit experimental IR value, but must not be exposed to -/// semantics-preserving replacement search: no error bound is -/// modeled for it (see [`HydraParams::HydraKll`]'s own doc). Enable automatic -/// selection only alongside an actual proof and error model. -/// -/// Not yet modeled: `HydraUnivMon`. The paper's own named "Hydra-sketch" is -/// really the universal-sketch composition (L layers of Count-Sketch plus a -/// heavy-hitter heap, Theorems 1+2 combined) estimating entropy/L1-norm/ -/// L2-norm/cardinality/frequency-moments as one instance. Standalone UnivMon -/// and its frequency evaluations are represented here, but sharing a Hydra -/// grid across populations still needs its own collision/error contract; -/// standalone support does not establish that contract. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -pub enum HydraKind { - /// Hydra over a KLL-family quantile sketch. See this type's own doc: - /// **not** an instance of the paper's proven construction — the paper - /// excludes quantiles from Hydra-sketch entirely. Kept only as an - /// explicit experimental representation; automatic replacement search - /// must not treat it as equivalent to independent instances. - HydraKll, - /// Hydra over Count-Min Sketch: a direct instance of the paper's proven - /// w×r shared-grid construction (§4.2/§4.5) — CMS is exactly the - /// linear frequency-vector substrate Theorem 2 is proved over. - HydraCms, - /// Hydra over Count-Sketch — the paper's own heavy-hitter substrate - /// (§4.3). A direct instance of the same proven construction as - /// `HydraCms`, with Count-Sketch's balanced/zero-mean-error trade - /// instead of CMS's one-sided bias. - HydraCountSketch, -} - -/// Parameters for a [`HydraKind`] instance. Unlike a plain [`SketchParams`] -/// (sized purely for one instance's own accuracy), a Hydra structure needs -/// both the per-subpopulation accuracy target it emulates *and* how big the -/// one shared structure itself is — the latter is the whole memory/accuracy -/// trade Hydra makes, and has no equivalent for an instance built -/// independently per subpopulation. One variant per [`HydraKind`] because -/// the knobs a "sketch of sketches" needs are specific to the inner -/// sketch's own parameter shape — a `HydraCms` instance is sized in -/// (`width`, `depth`), a `HydraKll` instance in `k`; there is no single knob -/// set general enough to cover every inner sketch type. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum HydraParams { - /// See [`HydraKind::HydraKll`]: kept for legality, **no accuracy bound - /// is modeled for this variant**. `k`/`shared_buckets` give a bound - /// sketch a concrete size to build against; unlike `HydraCms`/ - /// `HydraCountSketch`'s fields, they are not backed by the paper's - /// Theorem 2. - HydraKll { - /// Per-subpopulation accuracy knob — same meaning as - /// `SketchParams::Kll`'s own `k`, for the per-subpopulation logical - /// view this shared structure emulates. - k: u32, - /// Sizing knob for the one physical structure shared across every - /// subpopulation. Correctly sizing this against an estimated - /// subpopulation cardinality is a cost-model concern — out of scope - /// for the legality axis this type lives on — so this is - /// deliberately not derived from any cardinality estimate here. - shared_buckets: u32, - }, - /// Hydra over Count-Min Sketch — the paper's proven w×r - /// shared-grid construction (§4.2, Theorem 2). `width`/`depth` are the - /// per-subpopulation CMS knobs (mirroring `SketchParams::Cms`'s own - /// fields); `shared_rows`/`shared_columns` are the paper's `r`/`w` — the - /// redundant hash rows and shared-bucket width of the one physical grid - /// every subpopulation is hashed into. - HydraCms { - width: u32, - depth: u32, - /// The paper's `r`: redundant, pairwise-independent hash rows — - /// query time takes the median across rows to tighten the failure - /// probability (Theorem 2's `δ` term). - shared_rows: u32, - /// The paper's `w`: shared buckets per row that colliding - /// subpopulations share — the memory/accuracy knob Theorem 2's `ε` - /// term depends on. Sizing this against an estimated subpopulation - /// cardinality is a cost-model concern, deliberately out of scope - /// for the legality axis this type lives on. - shared_columns: u32, - }, - /// Hydra over Count-Sketch — same shape, and the same Theorem 2, as - /// `HydraCms`, over `SketchParams::CountSketch`'s knobs instead. - HydraCountSketch { - width: u32, - depth: u32, - shared_rows: u32, - shared_columns: u32, - }, -} - -/// How a grouped aggregate's summary state is physically instantiated -/// across its `by` subpopulations — orthogonal to *which* -/// `SketchKind`/`SamplingKind`/`WaveletKind`/`StatModelKind` answers the -/// intent (that choice lives alongside it on `FieldDataType`). Lives here, -/// alongside `SketchKind`/`SketchParams` etc., rather than on any of those -/// enums themselves, for exactly the reason explained in this section's -/// module docs above. -/// -/// Carried both on `ASAPOp::SummaryAgg` (where planning consults it) -/// and on sketch-valued `FieldDataType` edges (where it prevents -/// incompatible shared and independent physical states from type-checking -/// as merge-compatible). -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum GroupingStrategy { - /// One independent summary instance per distinct `by` key — today's - /// only (implicit) behavior, and this type's `Default` (below), so - /// every existing caller that never chose this axis explicitly keeps - /// observing exactly the same behavior it always has. - PerSubpopulationInstance, - /// One shared structure serving every subpopulation (Hydra and its - /// per-family variants — see [`HydraKind`]), trading per-subpopulation - /// isolation for shared memory/build cost. - /// - /// Named `SharedMultiSubpopulation`, not `SharedMultiTenant`: this - /// shares across a *query's own* subpopulations/group-by keys, not - /// across tenants in a deployment-isolation sense — a different, and - /// unrelated, kind of "sharing". - SharedMultiSubpopulation { - kind: HydraKind, - params: HydraParams, - }, -} - -impl Default for GroupingStrategy { - /// [`GroupingStrategy::PerSubpopulationInstance`] — see that variant's - /// own doc for why this is the only sound default. - fn default() -> Self { - Self::PerSubpopulationInstance - } -} - -// ── Sketch evaluation queries ───────────────────────────────────────────────── - -/// One summary-state update. `item` identifies a keyed item when the family -/// is keyed; `weight` is the observation/update expression applied to state. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct SummaryUpdate { - pub item: Option, - pub weight: SummaryInputExpr, - /// Planner proof for algorithms whose error/update semantics require a - /// non-negative stream. Absence of proof is never interpreted as zero or - /// non-negative. - #[serde(default)] - pub weight_domain: WeightDomain, -} - -impl SummaryUpdate { - pub fn column(column: ColumnRef) -> Self { - Self { - item: None, - weight: SummaryInputExpr::Column(column), - weight_domain: WeightDomain::UnknownOrSigned, - } - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum WeightDomain { - #[default] - UnknownOrSigned, - NonNegative { - proof: NonNegativeWeightProof, - }, - /// An open requirement, not a proof: the weights are raw data samples of - /// `metric`, or plain sums of them (no rate, no other arithmetic), so they - /// are non-negative when every raw data sample of `metric` is: `metric` is - /// declared a counter, or declared non-negative. Logical optimization - /// records it without reading the data workload; plan selection confirms - /// it from the declared source facts - /// ([`SourceFacts::metric_type`](crate::workload::SourceFacts::metric_type), - /// [`SourceFacts::values_non_negative`](crate::workload::SourceFacts::values_non_negative)). - /// Unconfirmed, it counts as not proven. Serialized as - /// `samples_of_metric`. - #[serde(rename = "samples_of_metric")] - RawDataSamplesOfMetric { - metric: String, - }, - /// An open requirement, not a proof: the weights are reset-corrected - /// `rate`, `increase` or `irate` values of `metric`'s raw data samples, or plain - /// sums of them. Reset correction keeps them non-negative exactly when - /// the samples are, that is when `metric` is a counter; over a gauge a - /// negative sample can give a negative value. Plan selection confirms - /// it from the declared metric types, as - /// [`RawDataSamplesOfMetric`](Self::RawDataSamplesOfMetric). Unconfirmed, it - /// counts as not proven. - CounterDerivativeOfMetric { - metric: String, - }, -} - -impl WeightDomain { - /// The domain once the declared facts have answered an open requirement. - /// [`RawDataSamplesOfMetric`](Self::RawDataSamplesOfMetric) is proven when the - /// metric is a counter ([`NonNegativeWeightProof::CounterRawDataSamples`]) or else - /// declared non-negative - /// ([`NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples`]); - /// [`CounterDerivativeOfMetric`](Self::CounterDerivativeOfMetric) only - /// when it is a counter - /// ([`NonNegativeWeightProof::ResetAwareCounterDerivative`]). Otherwise - /// unknown. Any other domain is returned unchanged. - pub(crate) fn confirmed( - &self, - is_counter: impl Fn(&str) -> bool, - is_declared_non_negative: impl Fn(&str) -> bool, - ) -> WeightDomain { - match self { - WeightDomain::RawDataSamplesOfMetric { metric } if is_counter(metric) => { - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples, - } - } - WeightDomain::RawDataSamplesOfMetric { metric } if is_declared_non_negative(metric) => { - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples, - } - } - WeightDomain::CounterDerivativeOfMetric { metric } if is_counter(metric) => { - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::ResetAwareCounterDerivative, - } - } - WeightDomain::RawDataSamplesOfMetric { .. } - | WeightDomain::CounterDerivativeOfMetric { .. } => WeightDomain::UnknownOrSigned, - other => other.clone(), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum NonNegativeWeightProof { - UnitCount, - /// `rate`, `increase` or `irate` of a counter's samples, or plain sums - /// of them: reset correction over non-negative samples gives a - /// non-negative increase, and a rate divides it by a positive duration. - /// Plan selection gives it on confirming a - /// [`WeightDomain::CounterDerivativeOfMetric`] requirement. - ResetAwareCounterDerivative, - /// Raw data samples of a metric declared a counter - /// ([`PrometheusMetricType::Counter`](crate::workload::PrometheusMetricType::Counter)), or - /// plain sums of them: a counter sample is never negative. Plan selection - /// gives it on confirming a - /// [`WeightDomain::RawDataSamplesOfMetric`] requirement. Serialized as - /// `counter_samples`. - #[serde(rename = "counter_samples")] - CounterRawDataSamples, - /// Raw data samples of a metric declared non-negative - /// ([`SourceFacts::values_non_negative`](crate::workload::SourceFacts::values_non_negative)), - /// such as a gauge, or plain sums of them. Plan selection gives it on - /// confirming a [`WeightDomain::RawDataSamplesOfMetric`] requirement for a - /// metric not declared a counter. Serialized as - /// `declared_non_negative_samples`. - #[serde(rename = "declared_non_negative_samples")] - DeclaredNonNegativeRawDataSamples, - /// A constant weight greater than 0, such as `Constant(2.0)`: every row - /// adds the same positive amount, so no weight is negative. - PositiveConstant, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum EntityIdentity { - PromqlLabelSet { excluding: Vec }, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum SummaryInputExpr { - Constant(f64), - Column(ColumnRef), - Tuple(Vec), - EntityIdentity(EntityIdentity), -} - -/// The statistic computed from summary state by `SummaryEstimate`, for example -/// `Quantile { q: 0.99 }`. This is a result operation, not a workload query. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum SketchStatistic { - /// sqrt(sum_v frequency(v)^2), not the norm of numeric input values. - FrequencyL2, - /// Shannon entropy of the value-frequency distribution, in bits. - FrequencyEntropy, - /// Extract the value at quantile rank `q` ∈ (0, 1]. - Quantile { q: f64 }, - /// Estimated count / frequency. `key` names which column is being - /// queried (`ColumnRef::SampleValue` for the bare bucket total, with - /// `value: None`); a `Named`/`Qualified` `key` paired with - /// `value: Some(v)` is a per-item point lookup (e.g. - /// `count(cms_metric{item="checkout"})` — `key` is `item`, `value` is - /// `"checkout"`). `value` is carried here rather than resolved by the - /// `SummaryExecutor` from a `Filter` predicate because a summary - /// evaluation is read without access to the rest of the DAG. - PointCount { - key: ColumnRef, - value: Option, - }, - /// Estimated number of distinct elements. - Cardinality, - /// Read the top-k entries from a heap-backed summary. - TopK { k: usize }, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn keyed_summary_input_and_topk_evaluation_round_trip() { - for input in [ - SummaryUpdate { - item: Some(SummaryInputExpr::EntityIdentity( - EntityIdentity::PromqlLabelSet { excluding: vec![] }, - )), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }, - SummaryUpdate { - item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( - ColumnRef::Named("service".into()), - )])), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::UnknownOrSigned, - }, - ] { - let input_json = serde_json::to_string(&input).unwrap(); - assert_eq!( - serde_json::from_str::(&input_json).unwrap(), - input - ); - let query = SketchStatistic::TopK { k: 10 }; - let json = serde_json::to_string(&query).unwrap(); - assert_eq!( - serde_json::from_str::(&json).unwrap(), - query - ); - } - } - - #[test] - fn sketch_kind_keeps_a_valid_algorithm_and_params_pair() { - let kind = SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ); - assert_eq!(kind.algorithm(), &SketchAlgorithm::Kll); - } - - /// Two KLLs with the same `k` are the same sketch, whatever failure - /// probability each was sized for, so they compare equal and can be shared. - #[test] - fn kll_params_compare_equal_whatever_their_failure_probability() { - let kll = |k, failure_probability| SketchParams::Kll { - k, - failure_probability, - }; - assert_eq!(kll(331, Some(0.001)), kll(331, None)); - assert_eq!(kll(331, Some(0.001)), kll(331, Some(0.0001))); - assert_ne!(kll(331, None), kll(269, None)); - assert_ne!( - kll(331, None), - SketchParams::Kmv { - k: 331, - relative_error: None - } - ); - let family = |params| { - crate::ir::schema::FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, params), - GroupingStrategy::default(), - ) - }; - assert_eq!(family(kll(331, Some(0.001))), family(kll(331, None))); - } - - #[test] - #[should_panic(expected = "SketchKind parameter mismatch")] - fn sketch_kind_rejects_params_from_another_algorithm() { - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Hll { - precision: 14, - relative_error: None, - }, - ); - } - - #[test] - fn grouping_strategy_default_is_per_subpopulation_instance() { - // Existing/default behavior must stay `PerSubpopulationInstance` — - // this must not change any existing test's observed behavior. - assert_eq!( - GroupingStrategy::default(), - GroupingStrategy::PerSubpopulationInstance - ); - } - - /// The raw-data-sample and summary-sample renames keep their JSON names, - /// so committed plans and workloads still read. - #[test] - fn renamed_sample_terms_keep_their_wire_names() { - use crate::ir::schema::FieldDataType; - let json = |value: serde_json::Value| value.to_string(); - let raw = WeightDomain::RawDataSamplesOfMetric { - metric: "m".to_string(), - }; - assert_eq!( - serde_json::to_string(&raw).unwrap(), - json(serde_json::json!({"kind": "samples_of_metric", "metric": "m"})) - ); - for (proof, name) in [ - ( - NonNegativeWeightProof::CounterRawDataSamples, - "counter_samples", - ), - ( - NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples, - "declared_non_negative_samples", - ), - ] { - assert_eq!( - serde_json::to_string(&proof).unwrap(), - format!("\"{name}\"") - ); - } - let summary_sample = FieldDataType::SummarySample( - SamplingKind::Reservoir, - SamplingParams::Reservoir { size: 64 }, - ); - assert_eq!( - serde_json::to_string(&summary_sample).unwrap(), - json(serde_json::json!({"Sample": ["Reservoir", {"Reservoir": {"size": 64}}]})) - ); - } -} diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 1ad5c2186..13ffd59b2 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -1,9 +1,10 @@ //! Ranked-row output contract shared by logical TopK and its implementations. +use crate::ir::operator::input::OperatorInput; use crate::ir::operator::{Reduction, Source}; use crate::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use crate::ir::scalar::{ColumnRef, ProjectItem}; use crate::ir::schema::{ColumnId, DataType, FieldDataType, PROMQL_SERIES_IDENTITY}; -use crate::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; +use crate::ir::{ASAPOp, NonASAPOp, Operator, ScalarExpr}; /// The column a top-k ranks `child` by. Over a one-measure grouped /// aggregate (the SQL heavy-hitter shape and PromQL `topk(k, sum by ..)`) it @@ -12,17 +13,17 @@ use crate::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; /// `FinalizeExactAccumulator` or `SummaryEstimate`, under the `Project` /// restoring its names (a renaming) and a value `HAVING`'s `Filter`. Over /// per-series rows (PromQL `topk by (..) (k, rows)`) it is the sample value. -pub fn ranked_column(child: &OperatorNode) -> Result { - let mut node = child; +pub fn ranked_column(child: &I) -> Result { + let mut node = child.clone(); let reduction = loop { - match &node.operator { + match node.operator() { // Only a renaming keeps the measure last. Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) - if cols.len() == child.schema.fields.len() + if cols.len() == child.schema().fields.len() && cols .iter() .enumerate() - .all(|(i, item)| item.expr == ScalarExpr::Column(i)) => + .all(|(i, item)| matches!(item.expr, ScalarExpr::Column(c) if c == i)) => { node = child } @@ -49,34 +50,34 @@ pub fn ranked_column(child: &OperatorNode) -> Result { } }; match reduction { - Some(Reduction::Reduce(_)) => Ok(child.schema.fields.len() - 1), - _ => resolve_column_ref(&ColumnRef::SampleValue, &child.schema), + Some(Reduction::Reduce(_)) => Ok(child.schema().fields.len() - 1), + _ => resolve_column_ref(&ColumnRef::SampleValue, child.schema()), } } /// Ordered projection of ranked rows: partition keys, ranked value, then /// item identity. TopK retains rows; partition keys alone are not unique. -pub fn ranked_row_projection( - child: &OperatorNode, +pub(crate) fn ranked_row_projection( + child: &I, reduction: &Reduction, output_name: &str, -) -> Result, crate::ir::SchemaDerivationError> { +) -> Result>, crate::ir::SchemaDerivationError> { use crate::ir::SchemaDerivationError; let value = ranked_column(child) .map_err(|error| SchemaDerivationError::InvalidScalarSignature(error.to_string()))?; let keys = reduction.group_keys().map_or(&[][..], |keys| keys.keys()); for &index in keys { - if index >= child.schema.fields.len() { + if index >= child.schema().fields.len() { return Err(SchemaDerivationError::InvalidGroupByColumn( index, - child.schema.fields.len(), + child.schema().fields.len(), )); } } - let mut columns: Vec = keys + let mut columns: Vec> = keys .iter() .map(|&index| ProjectItem { - alias: Some(child.schema.fields[index].name.clone()), + alias: Some(child.schema().fields[index].name.clone()), expr: ScalarExpr::Column(index), }) .collect(); @@ -85,7 +86,7 @@ pub fn ranked_row_projection( expr: ScalarExpr::Column(value), }); let promql = reads_only_time_series(child); - for (index, field) in child.schema.fields.iter().enumerate() { + for (index, field) in child.schema().fields.iter().enumerate() { let names_item = !promql || field.dtype == FieldDataType::Plain(DataType::Utf8); if names_item && index != value @@ -102,7 +103,7 @@ pub fn ranked_row_projection( // so consumers' already-bound column positions remain valid. if let Ok(identity) = resolve_column_ref( &ColumnRef::Named(PROMQL_SERIES_IDENTITY.into()), - &child.schema, + child.schema(), ) { if !keys.contains(&identity) { columns.push(ProjectItem { @@ -119,8 +120,8 @@ pub fn ranked_row_projection( pub fn series_identity_item( schema: &crate::ir::schema::Schema, partitions: &[ColumnId], -) -> crate::ir::schema::SummaryInputExpr { - use crate::ir::schema::SummaryInputExpr; +) -> crate::summary_type::SummaryInputExpr { + use crate::summary_type::SummaryInputExpr; let mut items: Vec<_> = schema .fields .iter() @@ -142,12 +143,11 @@ pub fn series_identity_item( } } -fn reads_only_time_series(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), - _ => node - .children() - .iter() - .all(|child| reads_only_time_series(child)), +fn reads_only_time_series(node: &I) -> bool { + match node.operator() { + Operator::NonASAP(NonASAPOp::Scan { source, .. }) => { + matches!(source, Source::TimeSeries { .. }) + } + operator => operator.children().into_iter().all(reads_only_time_series), } } diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index b6e8c0157..c33162e00 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -3,27 +3,36 @@ //! //! - [`ir`] — the unified operator IR (#511): one operator DAG for logical //! and physical plans ([`ir::OperatorNode`]), arranged by IR section: -//! [`ir::operator`] (§1 operators and their parameter types), +//! [`ir::operator`] (§1 operators and their argument types), //! [`ir::scalar`] (§2.2 scalar expressions), [`ir::schema`] (§2.1 the -//! per-edge schema and the summary-state types it carries) and -//! [`ir::properties`] (§2.2–2.3 accuracy, guarantee, execution timing and -//! summary coverage), the flat DAG -//! form ([`ir::flat`]) and the physical ASAP DAG ([`ir::physical_export`]). +//! per-edge schema) and [`ir::properties`] (§2.3 execution timing and +//! summary coverage), and the flat DAG form ([`ir::flat`]). +//! - [`summary_type`] — each summary type's parameters and predefined +//! parameter lists, the statistics read from it and its capabilities. +//! - [`accuracy`] — accuracy targets, the error metrics they are stated in, +//! and accuracy guarantees. +//! - [`deployment_capabilities`] — what the deployment's executor can run and +//! keep ([`deployment_capabilities::DeploymentCapabilities`]), the deployment +//! input Stages 1–3 read. //! - [`workload`] — the planner inputs (#509): the query and data //! workloads and the parsed workload the query-language frontends produce //! ([`workload::parsed_workload`]). -//! - [`physical`] — Stage 2 decision data: the materialization of a -//! sub-DAG ([`physical::Materialization`]). //! - [`stages`] — the types one planning stage hands the next: the Stage 1 //! → Stage 2 memo of groups and implementation candidates -//! ([`stages::LogicalASAPMemoCandidates`]) and the Stage 2 → Stage 3 -//! physical candidates ([`stages::PhysicalASAPMemoCandidates`]). +//! ([`stages::LogicalASAPMemoCandidates`]), the Stage 2 → Stage 3 +//! physical candidates ([`stages::PhysicalASAPMemoCandidates`]) and their +//! physical properties ([`stages::PhysicalProperties`]: timing and lifetime). +//! - [`plan`] — Stage 3's output: the physical ASAP DAG +//! ([`plan::physical_asap_dag`]) and the materialized views it installs. //! //! No execution logic lives in this crate (issue #190). +pub mod accuracy; +pub mod deployment_capabilities; pub mod ir; -pub mod physical; +pub mod plan; pub mod serde_f64; pub mod stages; +pub mod summary_type; #[cfg(feature = "test-support")] pub mod test_support; pub mod workload; diff --git a/crates/types/src/physical/materialization.rs b/crates/types/src/physical/materialization.rs deleted file mode 100644 index e30e192b3..000000000 --- a/crates/types/src/physical/materialization.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! The materialization option of each sub-DAG (#509 §Materialization), the -//! Stage 2 decision a node's execution timing -//! ([`crate::ir::properties::ExecutionTiming`]) is derived -//! from. - -use crate::ir::properties::ExecutionTiming; - -/// Where a materialized output is stored (#509 §Materialization). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -pub enum Medium { - Memory, - Disk, -} - -/// The materialization option of one sub-DAG (#509 §Materialization, -/// Algorithm 3): whether its output is stored across query executions, and -/// whether it is computed at ingestion time or at query time. -#[derive( - Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub enum Materialization { - /// Runs at query time for each execution; its output is discarded. - #[default] - NotMaterialized, - /// Runs as data arrives; its output is stored until a query reads it. - MatAtIngestion(Medium), - /// Runs when a query first needs it; its output is reused by the other - /// queries of the same evaluation and discarded after it. - MatAtQuery(Medium), - /// Runs at query time and keeps its output across evaluations: a - /// tumbling-window chain whose readers evaluate together every tumbling window - /// width builds its newest tumbling window at each evaluation and keeps the older - /// ones for the next (#509 Example 4's B3). Offered only when the - /// deployment keeps query-time results. - MatAtQueryKeptAcrossEvaluations(Medium), -} - -impl Materialization { - /// When the sub-DAG runs: only `MatAtIngestion` runs at ingestion time. - pub fn timing(self) -> ExecutionTiming { - match self { - Self::MatAtIngestion(_) => ExecutionTiming::IngestionTime, - Self::NotMaterialized - | Self::MatAtQuery(_) - | Self::MatAtQueryKeptAcrossEvaluations(_) => ExecutionTiming::QueryTime, - } - } - - /// Whether a node of this materialization is a materialized view: its - /// output is kept across evaluations. - pub(crate) fn is_materialized_view(self) -> bool { - matches!( - self, - Self::MatAtIngestion(_) | Self::MatAtQueryKeptAcrossEvaluations(_) - ) - } -} diff --git a/crates/types/src/physical/mod.rs b/crates/types/src/physical/mod.rs deleted file mode 100644 index bf10811ac..000000000 --- a/crates/types/src/physical/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! #509 Stage 2 decision data: the materialization of each sub-DAG. -//! -//! - [`materialization`] — whether a sub-DAG's output is stored and when it -//! is computed ([`Materialization`], [`Medium`]). -//! - [`workload_plan`] — one evaluated assignment rebuilt as a physical DAG -//! ([`PhysicalWorkloadPlan`]). -pub mod materialization; -pub mod workload_plan; - -pub use materialization::{Materialization, Medium}; -pub use workload_plan::PhysicalWorkloadPlan; diff --git a/crates/types/src/physical/workload_plan.rs b/crates/types/src/physical/workload_plan.rs deleted file mode 100644 index 47412bd9b..000000000 --- a/crates/types/src/physical/workload_plan.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! One evaluated materialization assignment rebuilt as a physical DAG. -use std::collections::BTreeMap; - -use crate::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; -use crate::ir::QueryRoot; - -use super::Materialization; - -/// One CandidatePhysicalASAPDAG (Stage 2 output), derived from exactly one -/// CandidateLogicalASAPDAG (Stage 1 output). It holds planning data only; -/// display labels are derived from it by the devtools. -#[derive(Debug, Clone)] -pub struct PhysicalWorkloadPlan { - /// `P` (nothing materialized) or `P-m`, set by the caller. - pub id: String, - /// The id of the logical candidate it implements. - pub from_logical: String, - /// The timed query roots, one per query in workload order, that `dag` - /// exports. - pub roots: Vec, - pub dag: PhysicalASAPDAG, - /// Its materialization assignment: the option chosen for each summary - /// build (`SummaryAgg` or `SummaryTumblingWindowChain`) of `dag`, by node id, including - /// the builds left `NotMaterialized`, and each other node materialized - /// (a shared node built once per evaluation). A build one query reads - /// at ingestion time and another at query time appears once per copy. - pub materialization: BTreeMap, -} - -impl PhysicalWorkloadPlan { - /// Whether any node is materialized: a summary build, or a shared - /// non-summary node built once per evaluation. - #[cfg(any(test, feature = "test-support"))] - pub fn materializes_any(&self) -> bool { - self.materialization - .values() - .any(|m| *m != Materialization::NotMaterialized) - } -} diff --git a/crates/types/src/ir/materialized_view.rs b/crates/types/src/plan/materialized_view.rs similarity index 74% rename from crates/types/src/ir/materialized_view.rs rename to crates/types/src/plan/materialized_view.rs index 6a3c2de29..dd8d51737 100644 --- a/crates/types/src/ir/materialized_view.rs +++ b/crates/types/src/plan/materialized_view.rs @@ -5,12 +5,12 @@ //! `retention_ms` installation sets on its nodes //! ([`PhysicalASAPDAG::set_retention`]). //! -//! A **materialized view** is a node whose output is kept across -//! evaluations: its materialization is `MatAtIngestion` or -//! `MatAtQueryKeptAcrossEvaluations`, and it carries its `retention_ms`. A -//! `MatAtQuery` node is not a view: its output is discarded after one +//! A **materialized view** is a node whose output is materialized across +//! evaluations: its lifetime is `AcrossEvaluations`, at ingestion time or +//! at query time, and it carries its `retention_ms`. A node materialized for +//! `OneEvaluation` is not a view: its output is discarded after one //! evaluation. A query-time `Scan` carries how long its source's rows are -//! kept for it (`None` when it reads them unbounded). +//! retained for it (`None` when it reads them unbounded). //! //! ## Refreshing a view //! @@ -69,24 +69,23 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; -use super::physical_export::{ - offset_below, PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, - PhysicalASAPNodeId, ScanWindow, +use super::physical_asap_dag::{ + PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, PhysicalASAPNodeId, }; +use crate::ir::operator::arguments::Source; use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::{Operator, OperatorNode}; use crate::ir::operator::non_asap::NonASAPOp; -use crate::ir::operator::operator_properties::Source; use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::summary_coverage::{CoverageError, SelectionBox, SummaryCoverage}; -use crate::physical::materialization::Materialization; +use crate::stages::Lifetime; /// One materialized view of a DAG: what its stored state is, what it is -/// refreshed from, when, and for how long it is kept. +/// refreshed from, when, and for how long it is retained. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct MaterializedView { - /// The node whose output is kept. + /// The node whose output is materialized. pub node_id: PhysicalASAPNodeId, /// Which run refreshes it. pub(crate) refresh: MaterializedViewRefresh, @@ -235,13 +234,13 @@ impl RawDataWindow { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum MaterializedViewRefresh { - /// Refreshed at ingestion time (`MatAtIngestion`): the runtime runs the + /// Refreshed at ingestion time (materialized at ingestion time): the runtime runs the /// node in an ingestion run once its inputs are complete (its windows' /// samples have arrived and the views it reads are refreshed), and /// keeps the result. AtIngestion, - /// Refreshed at query time and kept across evaluations - /// (`MatAtQueryKeptAcrossEvaluations`): when an evaluation reads this + /// Refreshed at query time and materialized across evaluations + /// (query-time `AcrossEvaluations`): when an evaluation reads this /// view for a selection it has not been refreshed for yet, the runtime /// runs the node in a query run over its inputs, keeps the result, and /// binds it like a view refreshed at ingestion. @@ -375,18 +374,18 @@ pub enum InstallableDagError { /// Bad input (a plan file): view `node_id` has no `retention_ms`. #[error("materialized view {node_id} has no retention_ms")] ViewWithoutRetention { node_id: PhysicalASAPNodeId }, - /// Bad input (a plan file): view `node_id` is kept for less than the + /// Bad input (a plan file): view `node_id` is retained for less than the /// data it summarizes reaches back, so a later evaluation would miss /// it. - #[error("materialized view {node_id} is kept for less than its data reaches back")] + #[error("materialized view {node_id} is retained for less than its data reaches back")] ViewRetentionTooShort { node_id: PhysicalASAPNodeId }, /// Bad input (a plan file): query-time `Scan` `node_id` reads a bounded /// window but has no `retention_ms`. #[error("query-time scan {node_id} reads a bounded window but has no retention_ms")] ScanWithoutRetention { node_id: PhysicalASAPNodeId }, /// Bad input (a plan file): query-time `Scan` `node_id` keeps its rows - /// for less than its readers' window reaches back. - #[error("query-time scan {node_id} is kept for less than its window reaches back")] + /// for less than its consumers' window reaches back. + #[error("query-time scan {node_id} is retained for less than its window reaches back")] ScanRetentionTooShort { node_id: PhysicalASAPNodeId }, } @@ -398,11 +397,9 @@ impl PhysicalASAPDAG { /// covers how far back its data reaches. O(views · nodes) for the /// coverages, plus O(nodes + edges). pub fn materialized_views(&self) -> Result, InstallableDagError> { - if let Some(node) = self - .nodes - .iter() - .find(|node| node.materialization.is_materialized_view() && node.retention_ms.is_none()) - { + if let Some(node) = self.nodes.iter().find(|node| { + matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) && node.retention_ms.is_none() + }) { return Err(InstallableDagError::ViewWithoutRetention { node_id: node.id }); } self.validate()?; @@ -511,8 +508,8 @@ impl PhysicalASAPDAG { /// `SummaryTumblingWindowChain` (a DAG with its chains unexpanded), its newest tumbling window's, /// `(tumbling_window_width, s mod tumbling_window_width)` with `s` the sum of the offsets of /// the `TimeShift`s below it; for any other node, that of its window - /// ([`Self::view_windows`], which counts a chain below it as all its - /// tumbling windows). A chain and each of its expanded tumbling windows have the same one. + /// (the windows below it, a chain below it counting as all its tumbling + /// windows). A chain and each of its expanded tumbling windows have the same one. /// `None` when the window is unbounded or empty. O(nodes + edges). pub fn refresh_window(&self, node: PhysicalASAPNodeId) -> Option { self.refresh_windows([node])[&node] @@ -524,45 +521,8 @@ impl PhysicalASAPDAG { &self, nodes: impl IntoIterator, ) -> HashMap> { - let payloads = self.payloads_by_id(); - let (chains, others): (Vec<_>, Vec<_>) = nodes.into_iter().partition(|id| { - matches!( - payloads.get(id), - Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) - ) - }); - let mut windows: HashMap> = self - .view_windows(others) - .into_iter() - .map(|(id, window)| { - let refresh_window = match window { - ScanWindow::Within { - end_offset_ms, - width_ms, - } => RefreshWindow::of_window(end_offset_ms, width_ms), - ScanWindow::Unbounded => None, - }; - (id, refresh_window) - }) - .collect(); - for chain in chains { - let Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child, - tumbling_window_width_ms, - .. - })) = payloads.get(&chain) - else { - continue; - }; - // The newest tumbling window ends at the offsets of the shifts below the chain. - let end_offset_ms = offset_below(*child, |id| payloads.get(&id).copied()); - let width_ms = i64::try_from(*tumbling_window_width_ms).unwrap_or(0); - windows.insert( - chain, - end_offset_ms.and_then(|offset| RefreshWindow::of_window(offset, width_ms)), - ); - } - windows + let _ = nodes.into_iter(); + todo!("#1207 PR 6: a node's refresh window from the TimeRange and TimeShift nodes above its inputs") } /// Sets `retention_ms` on every materialized view and every query-time @@ -571,95 +531,33 @@ impl PhysicalASAPDAG { /// /// A view takes `tumbling_window_retention_ms_by_node`'s value (Stage 3's retention /// of a tumbling window's chain: its offset plus its window plus the longest - /// range-evaluation span of its readers); otherwise how far back its - /// data reaches, plus, for a view built from other views, its readers' + /// range-evaluation span of its consumers); otherwise how far back its + /// data reaches, plus, for a view built from other views, its consumers' /// longest range-evaluation span (`reader_span_ms_by_node`). Then each - /// view is kept at least as long as every view built from it needs it: + /// view is retained at least as long as every view built from it needs it: /// as long as that view's own retention when it is refreshed on first /// read (a range request may refresh it for its oldest step), else as /// far back as that view's data reaches. This is each node's own /// requirement: a runtime that stores identical outputs of several /// nodes once keeps them for the largest requirement among those nodes. /// - /// A query-time `Scan` keeps its rows for its readers' window - /// ([`Self::scan_window`]'s `lookback_ms`) plus their longest + /// A query-time `Scan` keeps its rows for its consumers' window + /// (how far back they read) plus their longest /// range-evaluation span, and at least the retention of every view /// refreshed on first read from it (which may be refreshed for its /// oldest step); `None` when its window is unbounded. In an installed - /// DAG a chain kept across evaluations is its expanded tumbling windows, so its + /// DAG a chain materialized across evaluations is its expanded tumbling windows, so its /// scan keeps the reach of all of them: reusing a tumbling window refreshed at an /// earlier evaluation is optional, and a restart or an older range step - /// refreshes older tumbling windows from raw data. (Stage 3 prices a kept chain on + /// refreshes older tumbling windows from raw data. (Stage 3 prices a materialized chain on /// the unexpanded DAG as reading its newest tumbling window.) pub fn set_retention( &mut self, tumbling_window_retention_ms_by_node: &HashMap, reader_span_ms_by_node: &HashMap, ) -> Result, InstallableDagError> { - let (mut views, scans_read): (Vec, Vec>) = - self.views_with_node_retention()?.into_iter().unzip(); - let mut reach_ms_by_node = HashMap::new(); - let mut position_of = HashMap::new(); - for (position, view) in views.iter_mut().enumerate() { - let reach_ms = reach_ms_of(&view.inputs, &reach_ms_by_node); - let span_ms = match view.inputs.materialized_views.is_empty() { - true => 0, - false => reader_span_ms_by_node - .get(&view.node_id) - .copied() - .unwrap_or(0), - }; - view.retention_ms = tumbling_window_retention_ms_by_node - .get(&view.node_id) - .copied() - .unwrap_or(reach_ms + span_ms); - reach_ms_by_node.insert(view.node_id, reach_ms); - position_of.insert(view.node_id, position); - } - // Views before their inputs: an input stays while every view built - // from it may still be refreshed. - for index in (0..views.len()).rev() { - let view = &views[index]; - let needed_ms = match view.refresh { - MaterializedViewRefresh::OnFirstRead => view.retention_ms, - MaterializedViewRefresh::AtIngestion => reach_ms_by_node[&view.node_id], - }; - for input in view.inputs.materialized_views.clone() { - let kept = &mut views[position_of[&input]].retention_ms; - *kept = (*kept).max(needed_ms); - } - } - let mut retention_ms_by_node: HashMap> = views - .iter() - .map(|view| (view.node_id, Some(view.retention_ms))) - .collect(); - // How long each scan's rows are needed by the views refreshed on - // first read from it. - let mut view_need_ms_by_scan: HashMap = HashMap::new(); - for (view, scans) in views.iter().zip(&scans_read) { - if view.refresh == MaterializedViewRefresh::OnFirstRead { - for scan in scans { - let need_ms = view_need_ms_by_scan.entry(*scan).or_insert(0); - *need_ms = (*need_ms).max(view.retention_ms); - } - } - } - let scan_windows = self.query_time_scan_windows(); - for node in self.nodes.iter().filter(|node| is_query_time_scan(node)) { - let window_ms = scan_windows[&node.id].lookback_ms(); - let retention_ms = window_ms.map(|window_ms| { - let span_ms = reader_span_ms_by_node.get(&node.id).copied().unwrap_or(0); - let views_ms = view_need_ms_by_scan.get(&node.id).copied().unwrap_or(0); - (window_ms + span_ms).max(views_ms) - }); - retention_ms_by_node.insert(node.id, retention_ms); - } - for node in &mut self.nodes { - if let Some(&retention_ms) = retention_ms_by_node.get(&node.id) { - node.retention_ms = retention_ms; - } - } - Ok(views) + let _ = (tumbling_window_retention_ms_by_node, reader_span_ms_by_node); + todo!("#1207 PR 6: each materialized node's retention_ms is its h_v, which Stage 3 sets") } /// For each raw-data source some query-time `Scan` reads, in the order @@ -667,48 +565,10 @@ impl PhysicalASAPDAG { /// scans, `None` when one reads it unbounded: the raw data a runtime /// keeps for the DAG's queries, the raw-data counterpart of each view's /// retention. A source read only at ingestion time is absent. Checks - /// that each scan reading a bounded window ([`Self::scan_window`]) has a + /// that each scan reading a bounded window has a /// retention covering it. pub fn raw_data_retention(&self) -> Result, InstallableDagError> { - let scan_windows = self.query_time_scan_windows(); - let mut retention: Vec = Vec::new(); - for node in self.nodes.iter().filter(|node| is_query_time_scan(node)) { - let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload else { - continue; - }; - let window_ms = scan_windows[&node.id].lookback_ms(); - let lookback_ms = match (window_ms, node.retention_ms) { - (None, _) => None, - (Some(window_ms), Some(kept_ms)) if kept_ms >= window_ms => Some(kept_ms), - (Some(_), Some(_)) => { - return Err(InstallableDagError::ScanRetentionTooShort { node_id: node.id }) - } - (Some(_), None) => { - return Err(InstallableDagError::ScanWithoutRetention { node_id: node.id }) - } - }; - match retention.iter_mut().find(|kept| &kept.source == source) { - Some(kept) => { - kept.lookback_ms = kept.lookback_ms.zip(lookback_ms).map(|(a, b)| a.max(b)) - } - None => retention.push(RawDataRetention { - source: source.clone(), - lookback_ms, - }), - } - } - Ok(retention) - } - - /// [`Self::scan_window`] of each query-time `Scan`, with no chain kept, - /// by id. - fn query_time_scan_windows(&self) -> HashMap { - let scans = self - .nodes - .iter() - .filter(|node| is_query_time_scan(node)) - .map(|node| node.id); - self.scan_windows(scans, &HashSet::new()) + todo!("#1207 PR 6: each source's retention from its Scan nodes' retention_ms") } /// Checks that every raw-data window the DAG reads (each `TimeRange`, @@ -730,7 +590,7 @@ impl PhysicalASAPDAG { let views: HashSet = self .nodes .iter() - .filter(|node| node.materialization.is_materialized_view()) + .filter(|node| matches!(node.lifetime, Lifetime::AcrossEvaluations(_))) .map(|node| node.id) .collect(); // The IR of each node, which coverage is derived from. @@ -741,40 +601,20 @@ impl PhysicalASAPDAG { }; let mut derived = Vec::new(); let mut coverage_by_node: HashMap = HashMap::new(); - // The window each view summarizes ([`PhysicalASAPDAG::view_window`]): - // its raw-data windows and those of the views it reads, unioned. - let mut window_by_view: HashMap = HashMap::new(); for &id in index.children_first.iter().filter(|id| views.contains(id)) { let node = index.nodes[&id]; let (inputs, scans) = index.inputs_of_view(id, &views)?; - let read_windows: Vec = inputs - .materialized_views - .iter() - .filter_map(|read| window_by_view.get(read).copied()) - .collect(); - if let Some(window) = inputs - .raw_data_windows - .iter() - .map(|window| ScanWindow::Within { - end_offset_ms: window.end_offset_ms, - width_ms: window.width_ms, - }) - .chain(read_windows.iter().copied()) - .reduce(ScanWindow::union) - { - window_by_view.insert(id, window); - } - if matches!(node.materialization, Materialization::MatAtIngestion(_)) { + let refreshed_at_ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; + if refreshed_at_ingestion { // An ingestion run reads each view for the window ending at // its end, so a view read must end where the raw windows do. + if !inputs.materialized_views.is_empty() { + todo!("#1207 PR 6: the window a materialized view it reads ends at, from that node's time constraints") + } let mut end_offsets_ms: Vec = inputs .raw_data_windows .iter() .map(|window| window.end_offset_ms) - .chain(read_windows.iter().filter_map(|window| match window { - ScanWindow::Within { end_offset_ms, .. } => Some(*end_offset_ms), - ScanWindow::Unbounded => None, - })) .collect(); end_offsets_ms.sort_unstable(); end_offsets_ms.dedup(); @@ -796,9 +636,10 @@ impl PhysicalASAPDAG { coverage_by_node.insert(id, coverage.clone()); let view = MaterializedView { node_id: id, - refresh: match node.materialization { - Materialization::MatAtIngestion(_) => MaterializedViewRefresh::AtIngestion, - _ => MaterializedViewRefresh::OnFirstRead, + refresh: if refreshed_at_ingestion { + MaterializedViewRefresh::AtIngestion + } else { + MaterializedViewRefresh::OnFirstRead }, coverage: ViewCoverage { definition_id: coverage.definition_id(), @@ -817,7 +658,7 @@ impl PhysicalASAPDAG { /// `inputs` summarizes reaches: the farthest of its raw-data windows and of /// the views it reads (`reach_ms_by_node`, each already computed). /// Whether `node` is a `Scan` run at query time, which carries how long its -/// source's rows are kept for it. +/// source's rows are retained for it. pub(crate) fn is_query_time_scan(node: &PhysicalASAPDAGNode) -> bool { matches!(node.payload, Operator::NonASAP(NonASAPOp::Scan { .. })) && node.output_state.timing == ExecutionTiming::QueryTime @@ -840,8 +681,8 @@ fn reach_ms_of( /// The nodes of a DAG by id, each node's inputs in edge order, and two /// facts every raw-data window check reads, computed in one pass each over -/// the DAG (O(nodes + edges)). `scan_window` counts the windows of a `Scan` -/// for pricing; this walks down from each window to check its shape. +/// the DAG (O(nodes + edges)); it walks down from each window to check its +/// shape. struct ViewIndex<'a> { dag: &'a PhysicalASAPDAG, nodes: HashMap, @@ -1083,79 +924,17 @@ impl<'a> ViewIndex<'a> { } } -/// The coverage of view `id` (IR `ir`), refreshed from `inputs`. A merge's -/// is the union of its inputs' selections under their shared definition -/// ([`SummaryCoverage::derive`]). A summary built from exactly one view and -/// nothing else, with no selection of its own (a rollup to a coarser -/// grouping, or across series), takes that input's selection: its -/// definition is its own sub-DAG with the input replaced by the input's -/// definition, so the rollups of equal inputs share one definition. -/// Anything else keeps its own derived coverage. +/// The coverage of view `id` (IR `ir`), refreshed from `inputs`: the +/// coverage its memo group stores, which the realized plan will carry. +/// Coverage is derived only in the memo, never on node trees. fn coverage_of( id: PhysicalASAPNodeId, inputs: &MaterializedViewInputs, ir: &HashMap>, coverage_by_node: &HashMap, ) -> Result { - let node = &ir[&id]; - let own = SummaryCoverage::derive(node)?; - let [input] = inputs.materialized_views.as_slice() else { - return Ok(own); - }; - if !inputs.raw_data_windows.is_empty() - || matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. })) - || own.selection != [SelectionBox::default()] - { - return Ok(own); - } - let input_coverage = &coverage_by_node[input]; - let over_definition = with_replaced(node, &ir[input], &input_coverage.definition); - Ok(SummaryCoverage { - definition: SummaryCoverage::derive(&over_definition)?.definition, - selection: input_coverage.selection.clone(), - }) -} - -/// `node` with every use of `old` (by address) replaced by `new`; nodes -/// that do not reach `old` are shared, not copied. -fn with_replaced( - node: &Rc, - old: &Rc, - new: &Rc, -) -> Rc { - fn replace( - node: &Rc, - old: &Rc, - new: &Rc, - cache: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if Rc::ptr_eq(node, old) { - return Rc::clone(new); - } - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Rc::clone(done); - } - let children = node.children(); - let replaced: Vec> = children - .iter() - .map(|child| replace(child, old, new, cache)) - .collect(); - let result = if children - .iter() - .zip(&replaced) - .all(|(a, b)| Rc::ptr_eq(a, b)) - { - Rc::clone(node) - } else { - let mut next = replaced.into_iter(); - let mut copy = (**node).clone(); - copy.operator = node - .operator - .map_children(|_| next.next().expect("one per child")); - Rc::new(copy) - }; - cache.insert(Rc::as_ptr(node), Rc::clone(&result)); - result - } - replace(node, old, new, &mut HashMap::new()) + let _ = (id, inputs, ir, coverage_by_node); + todo!( + "#1207 PR 6: the view's coverage from its memo group's GroupProperties::aggregate_coverage" + ) } diff --git a/crates/types/src/plan/mod.rs b/crates/types/src/plan/mod.rs new file mode 100644 index 000000000..c5e16eae8 --- /dev/null +++ b/crates/types/src/plan/mod.rs @@ -0,0 +1,5 @@ +//! Stage 3's output: the plan, one physical ASAP DAG +//! ([`physical_asap_dag`]), and the materialized views it installs +//! ([`materialized_view`]). +pub mod materialized_view; +pub mod physical_asap_dag; diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs new file mode 100644 index 000000000..ebb2d33ed --- /dev/null +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -0,0 +1,653 @@ +//! Physical ASAP DAG (planner-layering stage 2 output). +//! +//! The flattened operators of [`crate::ir::flat`], plus the lifetime and +//! execution timing (data state) of every node and edge: the only plan +//! structure, in memory and, serialized, in the plan file. Stage 3 builds it +//! directly as its last step, one node per selected execution, each with the +//! timing and lifetime of the physical candidate it was built from. +//! The deployment executes the plan as given +//! (#509 Assumption 8): it does not change lifetime or timing. + +use std::collections::{HashMap, HashSet}; +use std::rc::Rc; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::ir::flat::NodeId; +use crate::ir::operator::asap::ASAPOp; +use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::non_asap::NonASAPOp; +use crate::ir::scalar::ScalarExpr; + +use crate::accuracy::AccuracyGuarantee; +use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionTiming}; +use crate::ir::schema::Schema; +use crate::stages::PhysicalCandidateId; +use crate::stages::{Lifetime, PhysicalPropertiesError}; +use crate::summary_type::SketchParams; + +/// A node's operator, with each child replaced by its node id. +pub type PhysicalASAPOperatorPayload = Operator; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum EdgeRole { + Input, + Left, + Right, + /// The consumer reads the producer from inside one of its scalar + /// expressions (`scalar(v)`, a scalar subquery, `EXISTS`, `IN`). + ScalarRef, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum GroupingEdgeCompatibility { + Identical, + ConsumerCoarsensProducer, + Incompatible, + NotApplicable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum WindowEdgeCompatibility { + /// Physical lowering must prove equal tumbling-window and query phases or install an + /// exact boundary residual. The logical DAG alone cannot make that claim. + #[serde(rename = "RequiresAlignedPanePhaseOrExactBoundaryResidual")] + RequiresAlignedTumblingWindowPhaseOrExactWindowEdgeResidual, + NotApplicable, +} + +pub type PhysicalASAPNodeId = NodeId; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhysicalASAPDAGNode { + pub id: PhysicalASAPNodeId, + pub payload: PhysicalASAPOperatorPayload, + /// Phase is a placement choice for every operator, independent of payload kind. + pub output_state: ExecutionDataState, + /// How long and where the deployment materializes this node's output: the + /// lifetime the physical properties assignment chose for the sub-DAG this + /// node produces. Any node may be materialized (#509 §Materialization); a node + /// with no assignment has lifetime `None`. With `output_state.timing` it + /// is the node's `PhysicalProperties`. + pub lifetime: Lifetime, + /// θ of a summary build (`SummaryAgg`, `SummaryTumblingWindowChain`): its + /// candidate's `LogicalProperties::parameters`, which the executor builds + /// the sketch from. `None` on every other node and for an exact build. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parameters: Option, + pub output_schema: Schema, + pub guarantee: Option, + /// For a materialized view (a node whose lifetime materializes its output + /// across evaluations), how far back from the newest evaluation + /// time its output must stay available once refreshed; for a query-time + /// `Scan`, how far back its source's rows must stay available (`None` + /// when it reads them unbounded). `None` on every other node, and on + /// every node of a DAG exported before installation sets it + /// ([`PhysicalASAPDAG::set_retention`]). A + /// node field rather than part of [`Lifetime`], because Stage 2 + /// enumerates lifetimes before Stage 3 knows retention. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retention_ms: Option, + /// The physical candidate the node was built from: in-memory provenance + /// for validation and tools, not part of the plan file; `None` for a plan + /// built by a custom pass. + #[serde(skip)] + pub selected_physical_candidate: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhysicalASAPDAGEdge { + pub producer: PhysicalASAPNodeId, + pub consumer: PhysicalASAPNodeId, + pub role: EdgeRole, + pub(crate) intermediate_schema: Schema, + pub(crate) data_state: ExecutionDataState, + pub(crate) grouping: GroupingEdgeCompatibility, + pub(crate) window: WindowEdgeCompatibility, +} + +#[cfg(any(test, feature = "test-support"))] +impl PhysicalASAPDAGEdge { + pub fn new( + producer: PhysicalASAPNodeId, + consumer: PhysicalASAPNodeId, + role: EdgeRole, + intermediate_schema: Schema, + data_state: ExecutionDataState, + grouping: GroupingEdgeCompatibility, + window: WindowEdgeCompatibility, + ) -> Self { + Self { + producer, + consumer, + role, + intermediate_schema, + data_state, + grouping, + window, + } + } + + pub fn intermediate_schema(&self) -> &Schema { + &self.intermediate_schema + } + + /// For tests that change a compiled DAG's edge schema. + pub fn intermediate_schema_mut(&mut self) -> &mut Schema { + &mut self.intermediate_schema + } + + pub fn data_state(&self) -> ExecutionDataState { + self.data_state + } + + /// For tests that retime a compiled DAG's edges. + pub fn data_state_mut(&mut self) -> &mut ExecutionDataState { + &mut self.data_state + } + + pub fn grouping(&self) -> GroupingEdgeCompatibility { + self.grouping + } + + pub fn window(&self) -> WindowEdgeCompatibility { + self.window + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhysicalASAPDAG { + pub nodes: Vec, + pub edges: Vec, + /// Semantic workload roots. Physical query/precompute sinks are selected + /// downstream by the control plane. + /// One root per operator query of the batch, in workload order. + pub roots: Vec, + /// One expression per standalone scalar query, keyed by the query's + /// position in the batch, in increasing order. Its operator references + /// name nodes of this DAG. A standalone scalar query is not a node (#511 + /// §2.3), so query `i` of the batch is the scalar root keyed `i` when + /// there is one, and otherwise the next entry of `roots`. + pub scalar_roots: Vec<(usize, ScalarExpr)>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum PhysicalASAPDAGValidationError { + /// A node whose timing and lifetime cannot go together: a planner bug. + #[error("node {node:?}: {reason}")] + PhysicalProperties { + node: PhysicalASAPNodeId, + #[source] + reason: PhysicalPropertiesError, + }, + /// A summary build (or maintained population) that is not materialized but does + /// not run at query time: a planner bug. + #[error( + "node {node:?} is a summary build that is not materialized but does not run at query time" + )] + UnmaterializedSummaryBuildNotAtQueryTime { node: PhysicalASAPNodeId }, + #[error("ingestion node {consumer:?} depends on query node {producer:?}")] + QueryDependencyInIngestion { + producer: PhysicalASAPNodeId, + consumer: PhysicalASAPNodeId, + }, + #[error( + "node {0:?} has a retention_ms but is neither a materialized view nor a query-time scan" + )] + UnexpectedRetention(PhysicalASAPNodeId), + #[error("duplicate physical ASAP node id {0:?}")] + DuplicateNodeId(PhysicalASAPNodeId), + #[error("physical ASAP DAG root {0:?} does not name a node")] + MissingRoot(PhysicalASAPNodeId), + #[error("edge endpoint {0:?} does not name a node")] + MissingEdgeEndpoint(PhysicalASAPNodeId), + #[error("edge {producer:?}->{consumer:?} schema differs from producer output")] + EdgeSchemaMismatch { + producer: PhysicalASAPNodeId, + consumer: PhysicalASAPNodeId, + }, + #[error("edge {producer:?}->{consumer:?} data state differs from producer output")] + EdgeDataStateMismatch { + producer: PhysicalASAPNodeId, + consumer: PhysicalASAPNodeId, + }, + #[error("incoming edges of node {consumer:?} do not match its operator's inputs")] + EdgeRoleMismatch { consumer: PhysicalASAPNodeId }, + #[error("physical ASAP DAG has no query roots")] + NoRoots, + #[error("scalar root keyed {0} is out of order or outside the batch")] + ScalarRootPosition(usize), + #[error("physical ASAP DAG contains a cycle")] + Cycle, + #[error("physical ASAP node {0:?} is not reachable from the root")] + UnreachableNode(PhysicalASAPNodeId), + #[error( + "summary aggregate node {node:?} output schema does not contain its declared summary type" + )] + SummaryTypeSchemaMismatch { node: PhysicalASAPNodeId }, +} + +impl PhysicalASAPDAG { + /// The plan's query roots as IR, in entry order: each operator root's + /// node, or the standalone scalar query keyed by its entry. Shared nodes + /// stay shared. For tests that inspect a plan as operator trees. + #[cfg(any(test, feature = "test-support"))] + pub fn query_roots(&self) -> Result, PhysicalASAPDAGValidationError> { + use crate::ir::QueryRoot; + let nodes = self.operator_nodes()?; + let mut operator_roots = self.roots.iter(); + let count = self.roots.len() + self.scalar_roots.len(); + Ok((0..count) + .map( + |entry| match self.scalar_roots.iter().find(|(index, _)| *index == entry) { + Some((_, expr)) => { + QueryRoot::Scalar(expr.map_operator_refs(&mut |id| Rc::clone(&nodes[id]))) + } + None => QueryRoot::Operator(Rc::clone( + &nodes[operator_roots.next().expect("one operator root per entry")], + )), + }, + ) + .collect()) + } + + /// Every node of this DAG as IR, by node id, with its exported + /// guarantee (an operator node has no physical properties): what a caller derives + /// IR properties of a plan file's node from, such as its + /// [`SummaryCoverage`](crate::ir::properties::summary_coverage::SummaryCoverage). + /// Shared inputs stay shared. Validates the DAG first. + pub(crate) fn operator_nodes( + &self, + ) -> Result>, PhysicalASAPDAGValidationError> { + self.validate()?; + let by_id: HashMap = + self.nodes.iter().map(|node| (node.id, node)).collect(); + let mut rebuilt: HashMap> = HashMap::new(); + // Children first, without recursion: `validate` rejected cycles. + for node in &self.nodes { + let mut pending = vec![(node.id, false)]; + while let Some((id, children_done)) = pending.pop() { + if rebuilt.contains_key(&id) { + continue; + } + let node = by_id[&id]; + if !children_done { + pending.push((id, true)); + pending.extend(node.payload.children().into_iter().map(|c| (*c, false))); + continue; + } + let operator = node + .payload + .map_children(|child| Rc::clone(&rebuilt[child])); + let restored = OperatorNode::with_schema(operator, node.output_schema.clone()) + .with_guarantee(node.guarantee.clone()); + rebuilt.insert(id, Rc::new(restored)); + } + } + Ok(rebuilt) + } + + pub fn validate(&self) -> Result<(), PhysicalASAPDAGValidationError> { + let mut nodes = HashMap::new(); + for node in &self.nodes { + if nodes.insert(node.id, node).is_some() { + return Err(PhysicalASAPDAGValidationError::DuplicateNodeId(node.id)); + } + // A node's timing and lifetime must go together; a summary + // build that is not materialized runs at query time. Any other node that + // is not materialized runs when its consumer runs. + if node.output_state.timing == ExecutionTiming::IngestionTime + && matches!(node.lifetime, Lifetime::OneEvaluation(_)) + { + return Err(PhysicalASAPDAGValidationError::PhysicalProperties { + node: node.id, + reason: PhysicalPropertiesError::OneEvaluationAtIngestion, + }); + } + if is_materializable_state_output(&node.payload) + && node.output_state.timing == ExecutionTiming::IngestionTime + && node.lifetime == Lifetime::None + { + return Err( + PhysicalASAPDAGValidationError::UnmaterializedSummaryBuildNotAtQueryTime { + node: node.id, + }, + ); + } + if node.retention_ms.is_some() + && !matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) + && !super::materialized_view::is_query_time_scan(node) + { + return Err(PhysicalASAPDAGValidationError::UnexpectedRetention(node.id)); + } + if let Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + .. + }) = &node.payload + { + let mut found_family = false; + for field in &node.output_schema.fields { + if &field.dtype == family { + found_family = true; + } + } + if !found_family { + return Err(PhysicalASAPDAGValidationError::SummaryTypeSchemaMismatch { + node: node.id, + }); + } + } + } + if self.roots.is_empty() && self.scalar_roots.is_empty() { + return Err(PhysicalASAPDAGValidationError::NoRoots); + } + let queries = self.roots.len() + self.scalar_roots.len(); + let mut next = 0; + for (position, _) in &self.scalar_roots { + if *position < next || *position >= queries { + return Err(PhysicalASAPDAGValidationError::ScalarRootPosition( + *position, + )); + } + next = position + 1; + } + // Operator roots and the nodes scalar roots read. + let root_ids: Vec = self + .roots + .iter() + .copied() + .chain( + self.scalar_roots + .iter() + .flat_map(|(_, expr)| expr.operator_refs().into_iter().copied()), + ) + .collect(); + for root in &root_ids { + if !nodes.contains_key(root) { + return Err(PhysicalASAPDAGValidationError::MissingRoot(*root)); + } + } + // Each node's incoming edges are exactly its operator's inputs, with + // the role of each input's position, so a swapped `Left`/`Right` + // cannot reorder the inputs of a binary operator. + // Count input occurrences instead of scanning a vector for each edge. + // Tumbling-window merges can have tens of thousands of distinct inputs; linear + // searches made validation quadratic in that fan-in. Multiplicities + // still matter when the same producer occupies several input slots. + let mut expected: HashMap<_, HashMap<_, usize>> = self + .nodes + .iter() + .map(|node| { + let mut counts = HashMap::new(); + for input in edge_roles(&node.payload) { + *counts.entry(input).or_insert(0) += 1; + } + (node.id, counts) + }) + .collect(); + for edge in &self.edges { + let inputs = expected.get_mut(&edge.consumer).ok_or( + PhysicalASAPDAGValidationError::MissingEdgeEndpoint(edge.consumer), + )?; + let count = inputs + .get_mut(&(edge.producer, edge.role)) + .filter(|count| **count > 0) + .ok_or(PhysicalASAPDAGValidationError::EdgeRoleMismatch { + consumer: edge.consumer, + })?; + *count -= 1; + } + if let Some((consumer, _)) = expected + .iter() + .find(|(_, inputs)| inputs.values().any(|count| *count != 0)) + { + return Err(PhysicalASAPDAGValidationError::EdgeRoleMismatch { + consumer: *consumer, + }); + } + let mut children: HashMap> = HashMap::new(); + for edge in &self.edges { + let producer = nodes.get(&edge.producer).ok_or( + PhysicalASAPDAGValidationError::MissingEdgeEndpoint(edge.producer), + )?; + if !nodes.contains_key(&edge.consumer) { + return Err(PhysicalASAPDAGValidationError::MissingEdgeEndpoint( + edge.consumer, + )); + } + if producer.output_state.timing == ExecutionTiming::QueryTime + && nodes[&edge.consumer].output_state.timing == ExecutionTiming::IngestionTime + { + return Err(PhysicalASAPDAGValidationError::QueryDependencyInIngestion { + producer: edge.producer, + consumer: edge.consumer, + }); + } + if edge.intermediate_schema != producer.output_schema { + return Err(PhysicalASAPDAGValidationError::EdgeSchemaMismatch { + producer: edge.producer, + consumer: edge.consumer, + }); + } + if edge.data_state != producer.output_state { + return Err(PhysicalASAPDAGValidationError::EdgeDataStateMismatch { + producer: edge.producer, + consumer: edge.consumer, + }); + } + children + .entry(edge.consumer) + .or_default() + .push(edge.producer); + } + fn visit( + id: PhysicalASAPNodeId, + children: &HashMap>, + visiting: &mut HashSet, + visited: &mut HashSet, + ) -> bool { + if visited.contains(&id) { + return true; + } + if !visiting.insert(id) { + return false; + } + if children + .get(&id) + .into_iter() + .flatten() + .any(|child| !visit(*child, children, visiting, visited)) + { + return false; + } + visiting.remove(&id); + visited.insert(id); + true + } + let mut visited = HashSet::new(); + for root in &root_ids { + if !visit(*root, &children, &mut HashSet::new(), &mut visited) { + return Err(PhysicalASAPDAGValidationError::Cycle); + } + } + fn mark( + id: PhysicalASAPNodeId, + children: &HashMap>, + reachable: &mut HashSet, + ) { + if !reachable.insert(id) { + return; + } + for child in children.get(&id).into_iter().flatten() { + mark(*child, children, reachable); + } + } + let mut reachable = HashSet::new(); + for root in &root_ids { + mark(*root, &children, &mut reachable); + } + if let Some(id) = nodes.keys().find(|id| !reachable.contains(id)) { + return Err(PhysicalASAPDAGValidationError::UnreachableNode(*id)); + } + Ok(()) + } +} + +/// Each input of `operator` with its edge role: the operator inputs, then the +/// nodes read by its scalar expressions (the order of [`Operator::children`]). +/// A node referenced several times from the scalar expressions is one +/// `ScalarRef` input, as cost lowering reads it once; ids are per node +/// pointer, so this dedupes by pointer. +fn edge_roles(operator: &Operator) -> Vec<(C, EdgeRole)> { + use EdgeRole::*; + let inputs: Vec = match operator { + Operator::NonASAP(op) => match op { + NonASAPOp::Join { .. } | NonASAPOp::SetOp { .. } | NonASAPOp::BinaryOp { .. } => { + vec![Left, Right] + } + NonASAPOp::Concat { children, .. } => vec![Input; children.len()], + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::PromqlVectorFromScalar(_) => vec![], + _ => vec![Input], + }, + Operator::ASAP(op) => match op { + ASAPOp::SummarySubtract { .. } | ASAPOp::SummaryJoin { .. } => vec![Left, Right], + ASAPOp::SummaryMerge { children, .. } => vec![Input; children.len()], + _ => vec![Input], + }, + }; + let children = operator.children(); + let (direct, scalar_refs) = children.split_at(inputs.len()); + let mut edges: Vec<(C, EdgeRole)> = direct.iter().map(|c| **c).zip(inputs).collect(); + for producer in scalar_refs { + if !edges.contains(&(**producer, ScalarRef)) { + edges.push((**producer, ScalarRef)); + } + } + edges +} + +/// Whether `operator`'s output is a sub-DAG output that materialization may +/// store: a `SummaryAgg`, a `SummaryTumblingWindowChain` or a `MaintainPopulation`. +pub(crate) fn is_materializable_state_output(operator: &Operator) -> bool { + matches!( + operator, + Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::MaintainPopulation { .. } + ) + ) +} + +/// Test-only: `roots` exported as one DAG with every node at query time and +/// not materialized, and no θ (every `parameters` is `None`), with each +/// exported node's operator node by node id. Executor tests hand-build DAGs +/// this way; planning builds the plan in Stage 3. +#[cfg(any(test, feature = "test-support"))] +pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( + roots: &[crate::ir::QueryRoot], +) -> Result<(PhysicalASAPDAG, Vec>), PhysicalASAPDAGValidationError> { + use crate::ir::properties::execution_data_state::DataPrimitive; + use crate::ir::QueryRoot; + let (flat, nodes_by_id) = crate::ir::flat::flatten(roots); + let mut scalar_roots = Vec::new(); + let roots = flat + .roots + .into_iter() + .enumerate() + .filter_map(|(position, root)| match root { + QueryRoot::Operator(id) => Some(id), + QueryRoot::Scalar(expr) => { + scalar_roots.push((position, expr)); + None + } + }) + .collect(); + let mut nodes: Vec = Vec::with_capacity(flat.nodes.len()); + let mut edges = Vec::new(); + for (id, flat_node) in flat.nodes.into_iter().enumerate() { + let node = &nodes_by_id[id]; + let primitive = match &node.operator { + Operator::ASAP(op) if op.produced_state().is_some() => DataPrimitive::SummaryState, + Operator::ASAP(ASAPOp::MaintainPopulation { .. }) => DataPrimitive::SummaryState, + _ if node.result_kind == crate::ir::OperatorResultKind::State => { + DataPrimitive::SummaryState + } + _ => DataPrimitive::Raw, + }; + let output_state = ExecutionDataState { + timing: ExecutionTiming::QueryTime, + primitive, + }; + // Children come before their parents, so every producer is already in `nodes`. + for (producer, role) in edge_roles(&flat_node.operator) { + edges.push(PhysicalASAPDAGEdge { + producer, + consumer: id, + role, + intermediate_schema: nodes[producer].output_schema.clone(), + data_state: nodes[producer].output_state, + grouping: grouping_compatibility(&nodes[producer].payload, &flat_node.operator), + window: WindowEdgeCompatibility::NotApplicable, + }); + } + nodes.push(PhysicalASAPDAGNode { + id, + payload: flat_node.operator, + output_state, + lifetime: Lifetime::None, + parameters: None, + output_schema: flat_node.schema, + guarantee: flat_node.guarantee, + retention_ms: None, + selected_physical_candidate: None, + }); + } + let dag = PhysicalASAPDAG { + nodes, + edges, + roots, + scalar_roots, + }; + dag.validate()?; + Ok((dag, nodes_by_id)) +} + +#[cfg(any(test, feature = "test-support"))] +fn grouping_compatibility( + producer: &Operator, + consumer: &Operator, +) -> GroupingEdgeCompatibility { + use crate::ir::operator::arguments::Reduction; + let ( + Operator::ASAP(ASAPOp::SummaryAgg { + reduction: producer, + .. + }), + Operator::ASAP(ASAPOp::SummaryAgg { + reduction: consumer, + .. + }), + ) = (producer, consumer) + else { + return GroupingEdgeCompatibility::NotApplicable; + }; + match (producer, consumer) { + (p, c) if p == c => GroupingEdgeCompatibility::Identical, + (Reduction::PerEntity, Reduction::Reduce(_)) => { + GroupingEdgeCompatibility::ConsumerCoarsensProducer + } + (Reduction::Reduce(p), Reduction::Reduce(c)) + if !p.is_without() && !c.is_without() && c.iter().all(|key| p.contains(key)) => + { + GroupingEdgeCompatibility::ConsumerCoarsensProducer + } + _ => GroupingEdgeCompatibility::Incompatible, + } +} diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs deleted file mode 100644 index 8d18a64bd..000000000 --- a/crates/types/src/stages/candidates.rs +++ /dev/null @@ -1,112 +0,0 @@ -//! Implementation candidates attached to memo groups. - -use std::collections::HashSet; -use std::rc::Rc; - -use super::{MemoGroup, MemoGroupId}; -use crate::ir::{OperatorNode, QueryRoot}; - -/// The memo: the memo groups, each query's root over them, and each group's -/// candidate implementations. -#[derive(Debug, Clone, Default)] -pub struct Memo { - pub groups: Vec, - /// Each query's root over groups, by index in the workload (the - /// initial logical memo's query roots); groups added later are never a - /// query's root. - pub query_roots: Vec>, - /// Each group's implementation candidates, by group index. - pub candidates: Vec>, -} - -/// One way to implement a group: its node as Stage 2 implements it, over -/// child groups. -#[derive(Debug, Clone)] -pub struct CandidateForMemoGroup { - /// The groups a fused computation also implements, beyond the group - /// whose candidate it is (its owner, implicit from its position in - /// [`Memo::candidates`]); empty otherwise. - pub fused_groups: Vec, - /// The implementation's own node(s): its root node, whose descendants - /// down to [`Self::input_groups`] are its private nodes. - pub root_node: Rc, - /// The groups it reads, whose implementations are chosen separately. - pub input_groups: Vec, - /// How it was made: by Pass 1, or by which Pass 2 transformation. - pub origin: OriginOfCandidateForMemoGroup, - /// The number of equal shares of its measures' accuracy budget its - /// summaries are sized for: 1 for the whole budget, `n` when it is one - /// of `n` approximate inputs of a query (a ratio's operand). This positive - /// generation parameter is already encoded in concrete sketch parameters; - /// it is not an equality constraint on the selected number of readers or - /// approximate inputs. Recompose and validate accuracy at query roots. - /// Larger allocations may be valid but costlier than necessary. - pub accuracy_budget_parts: usize, -} - -/// Where an implementation candidate came from. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum OriginOfCandidateForMemoGroup { - /// Pass 1: exact, exact accumulator, or a summary family and size. - Pass1, - /// One build serving several reader groups (summary capability). - SharedBuild, - /// A summary readout factored from a composite implementation. - SummaryResult, - /// Canonical value preprocessing feeding a summary build. - SummaryInput, - /// Tumbling windows, or the newest tumbling windows of another group's chain. - TumblingWindows, - /// A coarse group regrouping a finer group. - Rollup, - /// A top-k read from another group's ranking. - TopKReuse, - /// Its metric's scans widened to a shared label set, read through a - /// `Project` (scan widening). - WidenedScans, -} - -impl Memo { - /// The groups, each after every group it or one of its candidates - /// reads, the lower index first among those ready: a dependency order for - /// candidate traversal, whatever order transformations appended - /// groups in. `None` when those reads form a cycle, a group reading - /// itself included. - pub fn topological_order(&self) -> Option> { - let count = self.groups.len(); - let mut reads: Vec> = vec![HashSet::new(); count]; - for (g, group) in self.groups.iter().enumerate() { - let candidates = self.candidates.get(g).map_or(&[][..], Vec::as_slice); - for child in group - .child_groups - .iter() - .chain(candidates.iter().flat_map(|c| &c.input_groups)) - { - // A group reading itself never becomes ready: a cycle. - reads[g].insert(child.0 as usize); - } - } - let mut readers: Vec> = vec![Vec::new(); count]; - let mut waiting: Vec = reads.iter().map(HashSet::len).collect(); - for (g, read) in reads.iter().enumerate() { - for &child in read { - readers[child].push(g); - } - } - let mut ready: std::collections::BinaryHeap> = (0..count) - .filter(|&g| waiting[g] == 0) - .map(std::cmp::Reverse) - .collect(); - let mut order = Vec::with_capacity(count); - while let Some(std::cmp::Reverse(g)) = ready.pop() { - order.push(MemoGroupId(g as u32)); - for &reader in &readers[g] { - waiting[reader] -= 1; - if waiting[reader] == 0 { - ready.push(std::cmp::Reverse(reader)); - } - } - } - (order.len() == count).then_some(order) - } -} diff --git a/crates/types/src/stages/logical/candidate.rs b/crates/types/src/stages/logical/candidate.rs new file mode 100644 index 000000000..61d59a103 --- /dev/null +++ b/crates/types/src/stages/logical/candidate.rs @@ -0,0 +1,113 @@ +//! Candidate scope: each group's candidates $v = (e_v, p^{\log}_v)$ +//! (concepts doc `memo-groups-properties-requirements.md`, section 3; +//! #1203). + +use crate::ir::Operator; +use crate::ir::SchemaDerivationError; +use crate::stages::{LogicalRequirements, MemoGroupId}; +use crate::summary_type::SketchParams; + +/// One candidate: position `index_in_group` in group `group`'s candidates +/// ([`MemoGroup::candidates`](crate::stages::MemoGroup::candidates)). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] +pub struct CandidateForMemoGroupId { + pub group: MemoGroupId, + pub index_in_group: usize, +} + +/// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. +#[derive(Debug, Clone)] +pub struct CandidateForMemoGroup { + /// $e_v$. + pub expression: CandidateExpression, + /// $p^{\log}_v$. + pub logical_properties: LogicalProperties, + /// $r^{\log}_{c,i}$, one per input group: per child of + /// `expression.operator`, in [`Operator::children`] order. Changed only + /// through [`Memo::set_candidate_input_requirements`](crate::stages::Memo::set_candidate_input_requirements). + pub input_requirements: Vec, + /// The groups a fused computation also implements, beyond the group + /// whose candidate it is; empty otherwise. + pub fused_groups: Vec, + /// Which rule added it. Not in the formulation: kept for + /// `stage_pipeline --text` and the viewer (approved 2026-11-05). + pub origin: OriginOfCandidateForMemoGroup, +} + +/// Which rule added a candidate. Nothing in planning reads it. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub enum OriginOfCandidateForMemoGroup { + /// Pass 1: exact, exact accumulator, or a summary type. + Pass1, + /// Canonical value preprocessing feeding a summary build. + SummaryInput, + /// A tumbling-window chain, or a merge reading one, of a summary state group. + TumblingWindows, + /// A coarse group regrouping a finer group. + Rollup, + /// A top-k read from another group's ranking. + TopKReuse, + /// Its metric's scans widened to a shared label set, read through a + /// `Project` (scan widening). + WidenedScans, +} + +/// $e_v = \operatorname{op}_v(G_{v,1}, \dots, G_{v,n})$: one operator, with +/// its arguments, whose children are groups. The input groups $G_{v,i}$ are +/// [`Operator::children`], in that order. +#[derive(Debug, Clone, PartialEq)] +pub struct CandidateExpression { + pub operator: Operator, +} + +/// A candidate [`Memo::add_candidate`](crate::stages::Memo::add_candidate) rejects; the memo is unchanged. +#[derive(Debug, thiserror::Error)] +pub enum CandidateError { + /// The candidate's expression fails the node rules (schema derivation, + /// input validation, merge check): a bug in the rule that built it. + #[error("the candidate's expression breaks the node rules: {0}")] + Schema(SchemaDerivationError), + /// Its output schema or output kind differs from the group's: it does + /// not compute the group's result (a bug in the rule that built it). + #[error("the candidate's output differs from group {group:?}'s")] + OutputDiffersFromGroup { group: MemoGroupId }, +} + +/// $p^{\log}_v = \boldsymbol\theta_v$. The summary type is the operator's +/// argument, not a property, and a tumbling-window chain's width is the +/// chain operator's argument. [`Self::new`] rejects parameters of another +/// algorithm, not in the algorithm's list, missing on a sketch build (a chain +/// included) or present on an exact operator. +#[derive(Debug, Clone)] +pub struct LogicalProperties { + /// $\boldsymbol\theta_v$, from the algorithm's predefined list; `None` for + /// an exact operator. + parameters: Option, +} + +impl LogicalProperties { + pub fn new( + expression: &CandidateExpression, + parameters: Option, + ) -> Result { + let _ = (expression, parameters); + todo!("#1207 PR 1: reject invalid parameters") + } + + /// $\boldsymbol\theta_v$. + pub fn parameters(&self) -> Option<&SketchParams> { + self.parameters.as_ref() + } +} + +/// Logical properties that do not fit their operator: bad input to +/// [`LogicalProperties::new`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum LogicalPropertiesError { + #[error("an exact operator has no summary parameters")] + ParametersOnExactOperator, + #[error("a sketch build needs parameters from its algorithm's list")] + SketchBuildWithoutParameters, + #[error("the parameters are not in their algorithm's predefined list")] + ParametersNotInList, +} diff --git a/crates/types/src/stages/logical/edge.rs b/crates/types/src/stages/logical/edge.rs new file mode 100644 index 000000000..4c9896618 --- /dev/null +++ b/crates/types/src/stages/logical/edge.rs @@ -0,0 +1,15 @@ +//! Edge scope: the logical edges $(u, c, i)$ with $\alpha^{\log}$ (#1201). + +use crate::stages::CandidateForMemoGroupId; + +/// $(u, c, i)$ over logical candidates: `producer_candidate_id` is a +/// candidate of $G_{c,i}$, the group input `input_index_of_consumer` of +/// `consumer_candidate_id` reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct LogicalEdge { + pub producer_candidate_id: CandidateForMemoGroupId, + pub consumer_candidate_id: CandidateForMemoGroupId, + pub input_index_of_consumer: usize, + /// $\alpha^{\log}_{u,c,i}$. + pub satisfies_logical_requirement: bool, +} diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs new file mode 100644 index 000000000..d00f3f0a6 --- /dev/null +++ b/crates/types/src/stages/logical/group.rs @@ -0,0 +1,152 @@ +//! Group scope: memo groups, the distinct logical sub-expressions of the +//! workload, and the properties every candidate of a group shares. + +use crate::ir::operator::input::OperatorInput; +use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::schema::Schema; +use crate::ir::ASAPOp; +use crate::ir::SchemaDerivationError; +use crate::ir::{Operator, OperatorResultKind}; +use crate::stages::{CandidateForMemoGroup, Memo}; + +/// A group, by index in the memo's groups. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] +pub struct MemoGroupId(pub u32); + +/// One distinct logical sub-expression of the workload: what it computes +/// and the output every candidate of it must produce. +#[derive(Debug, Clone)] +pub struct MemoGroup { + /// What it computes: one operator, with its arguments. Each + /// `MemoGroupId` in it is an input group, never this group's own id. θ + /// is no argument of an operator, so it is not part of a group. + pub expression: Operator, + /// Derived once when the group is created, from its input groups; never + /// changes. + pub group_properties: GroupProperties, + /// The groups that read it ([`Self::parent_groups`]) through their + /// expression or a candidate's. + pub parents: Vec, + /// The queries whose roots reach it ([`Self::reached_by_queries`]), + /// ascending. + pub queries: Vec, + /// Its candidates: empty after Stage 0, filled by Stage 1. + pub candidates: Vec, +} + +impl MemoGroup { + /// The groups that read it. + pub fn parent_groups(&self) -> &[MemoGroupId] { + &self.parents + } + + /// The queries whose roots reach it, by index in the workload, ascending. + pub fn reached_by_queries(&self) -> &[usize] { + &self.queries + } +} + +/// What every candidate of a group produces, derived from the group's +/// expression and its input groups' properties by the rules that check +/// operator nodes. +#[derive(Debug, Clone)] +pub struct GroupProperties { + pub schema: Schema, + pub result_kind: OperatorResultKind, + /// Per output column, in schema order: the sign of its values. + pub column_sign: Vec, + /// Whether the result changes with the evaluation time: true for a + /// group under a `TimeRange` relative to the evaluation time + /// (`sum_over_time(m[5m])`), false for one under a fixed `@ T` or + /// reading no time. A result that does not depend on it is the same at + /// every evaluation, so a stored copy delivers on every grid. + pub depends_on_evaluation_time: bool, + /// What an aggregate group covers: a summary state, a chain, a merge + /// over a chain's windows, or an exact `Aggregate` result; `None` for + /// any other group. Derived today for summary states only; exact + /// `Aggregate` groups follow in #1207 PR 2. + pub aggregate_coverage: Option, + /// Whether two equal expressions return the same rows for the same + /// input rows, so they may be one group (not an unordered `LIMIT`). + pub shareable_when_equal: bool, +} + +/// The sign of one column's values, a group property derived from the +/// input groups like lineage (concepts doc, section 3, "Value sign"); a +/// `Scan`'s are declared per column in the data workload's source facts. +/// Error contracts that hold only for values of one sign read it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ValueSign { + NonNegative, + NonPositive, + Unknown, +} + +/// The properties of a group, or a candidate, computing `expression` over +/// its input groups: the rules that check operator nodes, reading each +/// input group's `group_properties` and `expression` +/// ([`MemoGroupAsInput`]), never a node tree. Fails when the expression +/// breaks them (schema derivation, input validation, merge check). +// `column_sign` and `depends_on_evaluation_time` are derived in #1207 PR 2; +// until then the result is unreachable. +#[allow(unreachable_code)] +pub(super) fn derive_group_properties( + memo: &Memo, + expression: &Operator, +) -> Result { + let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); + let schema = operator.output_schema()?; + operator.validate_inputs()?; + let aggregate_coverage = match SummaryCoverage::derive_from_inputs(&operator) { + Ok(coverage) => Some(coverage), + // A merge is valid only over states of one definition it may merge. + Err(error) if matches!(operator, Operator::ASAP(ASAPOp::SummaryMerge { .. })) => { + return Err(error.into()) + } + Err(_) => None, + }; + Ok(GroupProperties { + schema, + result_kind: operator.output_kind(), + aggregate_coverage, + shareable_when_equal: operator.is_shareable_when_equal(), + column_sign: todo!("#1207 PR 2: each output column's sign, per operator, from the input groups' and the sources' declared signs"), + depends_on_evaluation_time: todo!("#1207 PR 2: whether the result changes with the evaluation time, from the operator and the input groups"), + }) +} + +/// A memo group read as an operator's input: its `group_properties` and +/// its `expression`, whose inputs are read the same way. +#[derive(Clone, Copy)] +pub(crate) struct MemoGroupAsInput<'m> { + pub(crate) memo: &'m Memo, + pub(crate) group_id: MemoGroupId, +} + +impl<'m> MemoGroupAsInput<'m> { + fn group(&self) -> &'m MemoGroup { + &self.memo.groups()[self.group_id.0 as usize] + } +} + +impl OperatorInput for MemoGroupAsInput<'_> { + fn schema(&self) -> &Schema { + &self.group().group_properties.schema + } + + fn result_kind(&self) -> OperatorResultKind { + self.group().group_properties.result_kind + } + + fn operator(&self) -> Operator { + self.group().expression.map_children(|&group_id| Self { + memo: self.memo, + group_id, + }) + } + + fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { + self.group().group_properties.aggregate_coverage.as_ref() + } +} diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs new file mode 100644 index 000000000..efef26be1 --- /dev/null +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -0,0 +1,165 @@ +//! Memo scope: the memo, Stage 0's and Stage 1's one type, and each query +//! over its groups; the operations over the whole memo +//! (concepts doc `memo-groups-properties-requirements.md`, section 3; #1203). + +use std::collections::{BTreeSet, HashSet}; + +use super::group::derive_group_properties; +use crate::accuracy::AccuracyTarget; +use crate::ir::Operator; +use crate::ir::QueryRoot; +use crate::ir::SchemaDerivationError; +use crate::stages::{ + CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalRequirements, MemoGroup, + MemoGroupId, +}; + +/// The memo, Stage 0's and Stage 1's one type: the memo groups, each with +/// its candidates ([`MemoGroup::candidates`]: empty after Stage 0), and each +/// query's root over them. +#[derive(Debug, Clone, Default)] +pub struct Memo { + /// The groups, by id ([`Self::groups`]); added only through + /// [`Self::find_or_add_group`]. + groups: Vec, + /// The queries, by index in the workload. + pub queries: Vec, +} + +/// One query of the memo: its root over groups, its accuracy target and +/// $n_q$. +#[derive(Debug, Clone)] +pub struct MemoQuery { + pub root: QueryRoot, + /// $A_q$; `None` for a query without one (every query when accuracy is + /// not enforced). + pub accuracy_target: Option, + /// $n_q$, the number of values one evaluation returns; `None` when the + /// data workload does not bound it. Pass 2 turns a whole-result failure + /// probability into a per-value one with it. + pub number_of_values_per_query_evaluation: Option, +} + +impl Memo { + /// The groups, by id. + pub fn groups(&self) -> &[MemoGroup] { + &self.groups + } + + /// Adds `query` and returns its index: its root's groups and every + /// group below them list it ([`MemoGroup::reached_by_queries`]). Stage 0 + /// adds each query once. + pub fn add_query(&mut self, query: MemoQuery) -> usize { + let _ = query; + todo!("#1207 PR 2") + } + + /// Hash-consing: the group with this expression, added if new, with the + /// `GroupProperties` [`derive_group_properties`] derives. Fails when the + /// expression breaks the node rules (schema derivation, input + /// validation, merge check). + pub fn find_or_add_group( + &mut self, + expression: Operator, + ) -> Result { + let _group_properties = derive_group_properties(self, &expression)?; + todo!("#1207 PR 2: hash-consing over groups") + } + + /// Adds `candidate` to `group`'s candidates. Fails when its expression + /// breaks the node rules ([`derive_group_properties`]) or its output + /// schema or kind differs from the group's. + pub fn add_candidate( + &mut self, + group: MemoGroupId, + candidate: CandidateForMemoGroup, + ) -> Result { + let _output = derive_group_properties(self, &candidate.expression.operator) + .map_err(CandidateError::Schema)?; + let _ = group; + todo!("#1207 PR 2") + } + + /// The candidate `id`. + pub fn candidate(&self, id: CandidateForMemoGroupId) -> &CandidateForMemoGroup { + let _ = id; + todo!("#1207 PR 2") + } + + /// Replaces candidate `id`'s input requirements $r^{\log}_{c,i}$, one + /// per input group. The only way to change a candidate's $r^{\log}$: + /// Pass 2 sets a readout's required accuracy after the rules have run. + /// Panics when the vector does not have one entry per input group. + pub fn set_candidate_input_requirements( + &mut self, + id: CandidateForMemoGroupId, + input_requirements: Vec, + ) { + let candidate = &mut self.groups[id.group.0 as usize].candidates[id.index_in_group]; + assert_eq!( + input_requirements.len(), + candidate.expression.operator.children().len(), + "one input requirement per input group of candidate {id:?}" + ); + candidate.input_requirements = input_requirements; + } + + /// The groups, each after every group it or one of its candidates + /// reads, the lower index first among those ready: a dependency order for + /// candidate traversal, whatever order rules appended groups in. `None` + /// when those reads form a cycle, a group reading itself included. + pub fn topological_order(&self) -> Option> { + let count = self.groups.len(); + let mut reads: Vec> = vec![HashSet::new(); count]; + for (g, group) in self.groups.iter().enumerate() { + for child in group.expression.children().into_iter().chain( + group + .candidates + .iter() + .flat_map(|c| c.expression.operator.children()), + ) { + // A group reading itself never becomes ready: a cycle. + reads[g].insert(child.0 as usize); + } + } + let mut consumers: Vec> = vec![Vec::new(); count]; + let mut waiting: Vec = reads.iter().map(HashSet::len).collect(); + for (g, read) in reads.iter().enumerate() { + for &child in read { + consumers[child].push(g); + } + } + let mut ready: std::collections::BinaryHeap> = (0..count) + .filter(|&g| waiting[g] == 0) + .map(std::cmp::Reverse) + .collect(); + let mut order = Vec::with_capacity(count); + while let Some(std::cmp::Reverse(g)) = ready.pop() { + order.push(MemoGroupId(g as u32)); + for &consumer in &consumers[g] { + waiting[consumer] -= 1; + if waiting[consumer] == 0 { + ready.push(std::cmp::Reverse(consumer)); + } + } + } + (order.len() == count).then_some(order) + } +} + +/// The groups some query's root reads, ascending: the groups of the +/// `queries`' roots that are a query's result or a scalar query's operand. +pub fn root_groups(queries: &[MemoQuery]) -> BTreeSet { + queries + .iter() + .flat_map(|query| query.root.operator_refs()) + .copied() + .collect() +} + +/// The queries whose root reads `group`, ascending. +pub fn queries_rooted_at(queries: &[MemoQuery], group: MemoGroupId) -> Vec { + (0..queries.len()) + .filter(|&query| queries[query].root.operator_refs().contains(&&group)) + .collect() +} diff --git a/crates/types/src/stages/logical/memo_candidates.rs b/crates/types/src/stages/logical/memo_candidates.rs new file mode 100644 index 000000000..56f1d9232 --- /dev/null +++ b/crates/types/src/stages/logical/memo_candidates.rs @@ -0,0 +1,25 @@ +//! Stage 1, Pass 2's output, which Stage 2 expands: the memo after every +//! rule, the logical edges and the candidates dead-candidate elimination +//! removed. + +use crate::stages::{EvaluationGrid, LogicalEdge, Memo, PrunedLogicalCandidate}; + +/// Stage 1, Pass 2's output. +#[derive(Debug, Clone)] +pub struct LogicalASAPMemoCandidates { + /// The memo after the last rule. + pub memo: Memo, + /// Every $(u, c, i)$ with $u \in G_{c,i}$ between candidates that + /// survive dead-candidate elimination, grouped by $(c, i)$. + pub edges: Vec, + /// The candidates dead-candidate elimination removed: no logical edge + /// names them, and Stage 2 does not expand them. + pub pruned: Vec, + /// Per group, by `MemoGroupId`: the grid a result of the group is + /// evaluated on, the coarsest grid holding every evaluation time of the + /// queries reaching it (X = `sum_over_time(m[5m])` read every 60 s and + /// every 120 s: 60 s); `None` when those times are not on one known + /// grid. A result of the group materialized across evaluations is + /// stored at these times. + pub evaluation_grid_of_group: Vec>, +} diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs new file mode 100644 index 000000000..249ca41cc --- /dev/null +++ b/crates/types/src/stages/logical/mod.rs @@ -0,0 +1,20 @@ +//! The logical side: Stage 0's and Stage 1's memo, by scope (memo → group +//! → candidate → candidate input → edge → Stage 1's output). + +mod candidate; +mod edge; +mod group; +mod memo; +mod memo_candidates; +mod requirements; + +pub use candidate::{ + CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, + LogicalProperties, LogicalPropertiesError, OriginOfCandidateForMemoGroup, +}; +pub use edge::LogicalEdge; +pub(crate) use group::MemoGroupAsInput; +pub use group::{GroupProperties, MemoGroup, MemoGroupId, ValueSign}; +pub use memo::{queries_rooted_at, root_groups, Memo, MemoQuery}; +pub use memo_candidates::LogicalASAPMemoCandidates; +pub use requirements::{LogicalRequirements, RequiredAccuracy}; diff --git a/crates/types/src/stages/logical/requirements.rs b/crates/types/src/stages/logical/requirements.rs new file mode 100644 index 000000000..586112fdd --- /dev/null +++ b/crates/types/src/stages/logical/requirements.rs @@ -0,0 +1,29 @@ +//! Candidate-input scope: each candidate's input requirements +//! $r^{\log}_{c,i}$ (concepts doc `memo-groups-properties-requirements.md`, +//! section 2). + +use crate::accuracy::AccuracyMetric; + +/// $r^{\log}_{c,i}$: only what $G_{c,i}$'s identity does not fix. Only a +/// readout's input (`SummaryEstimate`, `FinalizeExactAccumulator`, +/// `EvaluatePopulation` over an approximate state) carries one (concepts +/// doc, section 2, "Logical requirements"). +#[derive(Debug, Clone, Default, PartialEq)] +pub struct LogicalRequirements { + /// `None` on every input that is not a readout's, and on a readout's + /// input when no accuracy target reaches it. + pub required_accuracy: Option, +} + +/// The accuracy a readout's input needs from its producer: the producer's +/// guarantee for the readout's statistic, evaluated at +/// `failure_probability`, is at most `max_error` in `metric`. It is also +/// the readout's own output error. +#[derive(Debug, Clone, Copy, PartialEq, serde::Serialize)] +pub struct RequiredAccuracy { + pub metric: AccuracyMetric, + /// $\varepsilon_r$, in the metric's units. + pub max_error: f64, + /// $\delta_r$, per value; 1 when the target sets none. + pub failure_probability: f64, +} diff --git a/crates/types/src/stages/logical_memo_candidates.rs b/crates/types/src/stages/logical_memo_candidates.rs deleted file mode 100644 index d9b4899c8..000000000 --- a/crates/types/src/stages/logical_memo_candidates.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Stage 1, Pass 2's output, which Stage 2 implements. - -use super::Memo; - -/// Stage 1, Pass 2's output: the Pass 1 groups plus the groups Pass 2 adds -/// (shared producers, shared summary states), each with its Pass 1 -/// candidates plus every Pass 2 rule's (by `origin`), including the -/// candidates reading a shared summary state. -#[derive(Debug, Clone)] -pub struct LogicalASAPMemoCandidates { - /// The expanded memo: groups, query roots (passed through from Pass 1) - /// and each group's candidates. - pub memo: Memo, -} diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs deleted file mode 100644 index 765ef09ca..000000000 --- a/crates/types/src/stages/memo.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! Memo groups: the distinct logical sub-expressions of the workload. - -use std::collections::{BTreeSet, HashMap}; -use std::rc::Rc; - -use crate::ir::{OperatorNode, QueryRoot}; - -/// A group, by index in the memo's groups. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct MemoGroupId(pub u32); - -/// One distinct logical sub-expression of the workload. -#[derive(Debug, Clone)] -pub struct MemoGroup { - /// Its logical node as the frontends (and Pass 1 targets) see it; its - /// children are other groups' nodes. - pub logical_node: Rc, - /// Its child groups, in the node's child order. - pub child_groups: Vec, - /// The groups that read it ([`Self::parent_groups`]). Public only so - /// Stage 1, which builds and extends the memo, can maintain it; later - /// stages read it through [`Self::parent_groups`]. - pub parents: Vec, - /// The queries whose roots reach it ([`Self::reached_by_queries`]). - /// Public only so Stage 1, which builds and extends the memo, can - /// maintain it; later stages read it through - /// [`Self::reached_by_queries`]. - pub queries: Vec, -} - -impl MemoGroup { - /// The groups that read it. - pub fn parent_groups(&self) -> &[MemoGroupId] { - &self.parents - } - - /// The queries whose roots reach it, by index in the workload, ascending. - pub fn reached_by_queries(&self) -> &[usize] { - &self.queries - } -} - -/// Stage 0's output, Pass 1's input: the workload's memo groups, one per -/// distinct logical sub-expression, without implementation choices or -/// solver state. Stage 1 builds it by hash-consing the query roots. -#[derive(Debug, Clone)] -pub struct DagOfLogicalMemoGroups { - pub groups: Vec, - /// Each query's root over groups, by index in the workload: its - /// operator's group, or its scalar expression with every operator - /// reference replaced by that operator's group. - pub query_roots: Vec>, - /// Each node of the original workload, by address, to its group. - group_of: HashMap<*const OperatorNode, MemoGroupId>, -} - -impl DagOfLogicalMemoGroups { - /// The memo groups `groups` (children before parents), each query's - /// root over them, and `group_of`: each node of the original workload, - /// by address, to its group. - pub fn new( - groups: Vec, - query_roots: Vec>, - group_of: HashMap<*const OperatorNode, MemoGroupId>, - ) -> Self { - Self { - groups, - query_roots, - group_of, - } - } - - /// Group of a node from the original workload, not a rewritten candidate. - pub fn group_for_node(&self, node: &Rc) -> Option { - self.group_of.get(&Rc::as_ptr(node)).copied() - } - - /// Each query's root over the groups' logical nodes, in workload order: - /// a group several queries read is one node. - pub fn roots_over_groups(&self) -> Vec { - let node = |g: &MemoGroupId| Rc::clone(&self.groups[g.0 as usize].logical_node); - self.query_roots - .iter() - .map(|root| match root { - QueryRoot::Operator(g) => QueryRoot::Operator(node(g)), - QueryRoot::Scalar(expr) => { - QueryRoot::Scalar(expr.map_operator_refs(&mut |g| node(g))) - } - }) - .collect() - } -} - -/// The groups some query's root reads, ascending: the groups of -/// `query_roots` that are a query's result or a scalar query's operand. -pub fn root_groups(query_roots: &[QueryRoot]) -> BTreeSet { - query_roots - .iter() - .flat_map(QueryRoot::operator_refs) - .copied() - .collect() -} - -/// The queries whose root reads `group`, ascending. -pub fn queries_rooted_at(query_roots: &[QueryRoot], group: MemoGroupId) -> Vec { - (0..query_roots.len()) - .filter(|&query| query_roots[query].operator_refs().contains(&&group)) - .collect() -} diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 109ce200e..fa095e9bc 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -1,35 +1,33 @@ -//! The Stage 0 → Stage 1 boundary: the workload's memo groups -//! ([`DagOfLogicalMemoGroups`]). Inside Stage 1, Pass 1 → Pass 2: the memo -//! groups with their Pass 1 candidates -//! ([`PerComputationLogicalMemoCandidates`]). The Stage 1 → Stage 2 boundary: the memo of logical groups and their +//! The Stage 0 → Stage 1 boundary: the workload's memo groups, without +//! candidates ([`Memo`]). Inside Stage 1, Pass 1 → Pass 2: the memo with +//! its Pass 1 candidates. The Stage 1 → Stage 2 boundary: the memo of logical groups and their //! implementation candidates that Stage 1 (Pass 1 and Pass 2) produces and //! Stage 2 implements (#1047, #1054). The Stage 2 → Stage 3 boundary: the //! physical candidates Stage 2 builds and Stage 3 selects from. Stage //! crates never depend on each other, so the types more than one stage reads //! live here. -mod candidates; -mod logical_memo_candidates; -mod memo; +mod logical; mod per_computation_logical_memo_candidates; mod physical; mod pruning; -mod realizer; -pub use candidates::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -pub use logical_memo_candidates::LogicalASAPMemoCandidates; -pub use memo::{queries_rooted_at, root_groups, DagOfLogicalMemoGroups, MemoGroup, MemoGroupId}; +pub(crate) use logical::MemoGroupAsInput; +pub use logical::{ + queries_rooted_at, root_groups, CandidateError, CandidateExpression, CandidateForMemoGroup, + CandidateForMemoGroupId, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, + LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupId, + MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, +}; pub use per_computation_logical_memo_candidates::{ - ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, - HowToComputeAggregateMeasure, PerComputationLogicalMemoCandidates, + ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, }; pub use physical::{ - output_references, CandidateForMemoGroupId, DagOfSelectedCandidates, - OperatorNodeOfCandidateForMemoGroup, OperatorNodeOfCandidateForMemoGroupId, - PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalCandidateKey, - ProducerCandidates, ProducerSetInterner, ReaderConstraint, SelectedEdge, SelectedNodeId, + output_references, producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, + ExecutionContextId, Lifetime, PhysicalASAPMemoCandidates, PhysicalCandidate, + PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, PhysicalProperties, PhysicalPropertiesError, + StorageMedium, +}; +pub use pruning::{ + PruneCause, PruneReason, PrunedCandidate, PrunedLogicalCandidate, RootPruneReason, }; -#[cfg(any(test, feature = "test-support"))] -pub use physical::{producer_storage_metrics, ProducerStorageMetrics}; -pub use pruning::{PruneReason, PrunedCandidate}; -pub use realizer::{AssignmentRealizer, PhysicalCandidateError, RealizedPlan}; diff --git a/crates/types/src/stages/per_computation_logical_memo_candidates.rs b/crates/types/src/stages/per_computation_logical_memo_candidates.rs index 307f4bdc4..5361b3f12 100644 --- a/crates/types/src/stages/per_computation_logical_memo_candidates.rs +++ b/crates/types/src/stages/per_computation_logical_memo_candidates.rs @@ -1,26 +1,10 @@ -//! Stage 1, Pass 1's output, which Pass 2 extends (the memo groups with -//! their Pass 1 candidates), and the computation-target types Pass 1 builds -//! those candidates from. +//! The computation-target types Stage 1, Pass 1 builds its candidates from. use std::rc::Rc; -use super::{Memo, MemoGroupId, PrunedCandidate}; -use crate::ir::schema::{ExactKind, SketchAlgorithm}; +use super::MemoGroupId; use crate::ir::{OperatorNode, Predicate}; - -/// Stage 1, Pass 1's output: the workload's memo groups, each with its -/// Pass 1 candidates, and the candidates Pass 1 pruned. It is also Pass 2's -/// input. -#[derive(Debug, Clone)] -pub struct PerComputationLogicalMemoCandidates { - /// The groups moved from the [`DagOfLogicalMemoGroups`](super::DagOfLogicalMemoGroups), - /// its query roots, and every group's candidates (origin - /// [`Pass1`](super::OriginOfCandidateForMemoGroup::Pass1)). - pub memo: Memo, - /// The ways to compute a target's measure that Pass 1 pruned, each on - /// its target's aggregate group, in pruning order. - pub pruned_candidates: Vec>, -} +use crate::summary_type::{ExactKind, SketchAlgorithm}; /// How one Pass 1 target is computed: exact or a summary. /// @@ -84,8 +68,6 @@ pub struct ComputationTarget { /// Index into the aggregate's measures. pub measure_index: usize, /// The measure's unranked ways to compute it, the exact option first. - /// The ones Pass 1 pruned are in - /// [`PerComputationLogicalMemoCandidates::pruned_candidates`]. pub ways_to_compute: Vec, /// The measure's own filter; only rows that pass it update the state /// (`SummaryAgg.filter`). @@ -114,10 +96,8 @@ pub struct FusedComputation { /// computations absorb. None of them is computed when this is picked. pub absorbed_targets: Vec, /// How the fused computation is computed: exact or a summary. Empty - /// when Pass 1 pruned every one (today only for - /// [`PruneReason::UnsupportedByDeployment`](super::PruneReason::UnsupportedByDeployment), - /// recorded in [`PerComputationLogicalMemoCandidates::pruned_candidates`]); the fusion is - /// then not offered. + /// when Pass 1 pruned every one (today only for a family the deployment + /// does not support); the fusion is then not offered. pub ways_to_compute: Vec, } diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs new file mode 100644 index 000000000..130817db2 --- /dev/null +++ b/crates/types/src/stages/physical/candidate.rs @@ -0,0 +1,53 @@ +//! Candidate scope: each logical candidate's physical alternatives, one per +//! valid $p^{\mathrm{phys}} = (\tau, \ell)$ (#1200). + +use crate::ir::properties::ExecutionTiming; +use crate::stages::PhysicalProperties; +use crate::stages::{CandidateForMemoGroupId, ExecutionContext, PhysicalASAPMemoCandidates}; + +/// Index into [`PhysicalASAPMemoCandidates::physical_candidates`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] +pub struct PhysicalCandidateId(pub usize); + +/// One physical alternative of a logical candidate: what Stage 3 selects. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PhysicalCandidate { + /// $e_v$ and $p^{\log}_v$. + pub logical_candidate_id: CandidateForMemoGroupId, + /// $p^{\mathrm{phys}}_v$. + pub physical_properties: PhysicalProperties, +} + +impl PhysicalCandidate { + /// Whether a consumer running in `context` can read this producer (the + /// concepts doc's alignment table, section 2). True for $\ell$ = None, + /// and for `OneEvaluation` in the same context. A chain producer + /// (`SummaryTumblingWindowChain`) with $\ell$ = `AcrossEvaluations`, with + /// $w$ the chain operator's width, needs every time of the context on + /// its grid: $w$ divides the context's period and phase. Any other + /// producer with $\ell$ = `AcrossEvaluations` is stored on its group's + /// evaluation grid (`evaluation_grid_of_group`): it delivers on every + /// context when the group does not depend on the evaluation time, else + /// when that grid holds every time of the context. In + /// `Materialized(v)`, the context's times are v's, read from + /// `candidates`. + pub fn delivers_on( + &self, + candidates: &PhysicalASAPMemoCandidates, + context: &ExecutionContext, + ) -> bool { + let _ = (candidates, context); + todo!("#1207 PR 1: the alignment table of section 2") + } +} + +/// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime only (the concepts +/// doc's table, section 2). Alignment is checked per context in Stage 3 with +/// [`PhysicalCandidate::delivers_on`]. +pub fn producer_satisfies_physical_requirements( + producer: &PhysicalCandidate, + consumer_timing: ExecutionTiming, +) -> bool { + let _ = (producer, consumer_timing); + todo!("#1207 PR 1: the timing x lifetime table of section 2") +} diff --git a/crates/types/src/stages/physical/edge.rs b/crates/types/src/stages/physical/edge.rs new file mode 100644 index 000000000..7df48ad9a --- /dev/null +++ b/crates/types/src/stages/physical/edge.rs @@ -0,0 +1,20 @@ +//! Edge scope: the physical edges $\mathcal E$ with $\alpha^{\log}$ and +//! $\alpha^{\mathrm{phys}}$ (#1201). + +use crate::stages::PhysicalCandidateId; + +/// Index into [`PhysicalASAPMemoCandidates::edges`](crate::stages::PhysicalASAPMemoCandidates::edges). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct PhysicalEdgeId(pub usize); + +/// $(u, c, i)$ over physical candidates. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PhysicalEdge { + pub producer_candidate_id: PhysicalCandidateId, + pub consumer_candidate_id: PhysicalCandidateId, + pub input_index_of_consumer: usize, + /// $\alpha^{\log}_{u,c,i}$. + pub satisfies_logical_requirement: bool, + /// $\alpha^{\mathrm{phys}}_{u,c,i}$. + pub satisfies_physical_requirement: bool, +} diff --git a/crates/types/src/stages/physical/execution_context.rs b/crates/types/src/stages/physical/execution_context.rs new file mode 100644 index 000000000..31f95c13f --- /dev/null +++ b/crates/types/src/stages/physical/execution_context.rs @@ -0,0 +1,34 @@ +//! $\kappa$: a class of executions that happen at the same times (concepts +//! doc, section 5). Stage 3 indexes $x_{\kappa,v}$ and $y_{\kappa,u,c,i}$ by +//! it; [`super::PhysicalCandidate::delivers_on`] reads it, so it lives with +//! the physical candidates. + +use super::candidate::PhysicalCandidateId; +use crate::plan::materialized_view::RefreshWindow; + +/// Index of an [`ExecutionContext`] in the workload's list of distinct +/// contexts. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ExecutionContextId(pub usize); + +/// $\kappa$: within one context a computation runs once, and every consumer +/// in it shares that run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ExecutionContext { + /// At every time of an evaluation grid, answering queries: every query + /// evaluated on the grid, a range query's steps included. An alert + /// every 10 s at phase 0 is the grid 10 s @ 0. + Evaluation { grid: EvaluationGrid }, + /// As data arrives, in batches of one refresh window. + IngestionPipeline(RefreshWindow), + /// The own computation of a candidate materialized across evaluations, + /// on its own schedule, whoever reads it. + Materialized(PhysicalCandidateId), +} + +/// The evaluation times `phase_ms + k · period_ms`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct EvaluationGrid { + pub period_ms: u64, + pub phase_ms: u64, +} diff --git a/crates/types/src/stages/physical/graph.rs b/crates/types/src/stages/physical/graph.rs deleted file mode 100644 index c4537eaa4..000000000 --- a/crates/types/src/stages/physical/graph.rs +++ /dev/null @@ -1,281 +0,0 @@ -//! Selections over Stage 2's physical candidates, and their structural -//! validation. -use super::memo_candidates::PhysicalASAPMemoCandidates; -use super::producers::ProducerCandidates; -use std::collections::{HashMap, HashSet}; - -/// Index into [`PhysicalASAPMemoCandidates::candidates`](super::PhysicalASAPMemoCandidates::candidates). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct PhysicalCandidateId(pub usize); - -/// A rule on which output references may read a candidate, applied to each -/// node of a [`DagOfSelectedCandidates`] that runs it: the output references -/// that reach the node are its readers. Output references -/// index the group references of the query roots -/// ([`output_references`](super::output_references)); several references of -/// one query stay distinct. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum ReaderConstraint { - /// Each node is read by at least `minimum` queries. Each bucket of - /// `query_outputs` is one query's output references; together they - /// partition every output exactly once. Empty buckets represent - /// literal-only queries. - MinimumQueryReaders { - candidate: PhysicalCandidateId, - query_outputs: Vec>, - minimum: usize, - }, - /// A non-summary query materialization: each node produces an - /// output reference, or has an immediate selected consumer reached by - /// at most one query (`query_outputs` as in `MinimumQueryReaders`). - HighestSharedQueryMaterialization { - candidate: PhysicalCandidateId, - query_outputs: Vec>, - }, - /// Only `outputs` may read a node. - AllowedOutputs { - candidate: PhysicalCandidateId, - outputs: Vec, - }, - /// No node is read by both `left` and `right`. - IncompatibleOutputs { - candidate: PhysicalCandidateId, - left: usize, - right: usize, - }, -} - -impl ReaderConstraint { - /// The candidate it constrains. - pub fn candidate(&self) -> PhysicalCandidateId { - match self { - Self::MinimumQueryReaders { candidate, .. } - | Self::HighestSharedQueryMaterialization { candidate, .. } - | Self::AllowedOutputs { candidate, .. } - | Self::IncompatibleOutputs { candidate, .. } => *candidate, - } - } - - /// The candidate it constrains, to renumber. - pub fn candidate_mut(&mut self) -> &mut PhysicalCandidateId { - match self { - Self::MinimumQueryReaders { candidate, .. } - | Self::HighestSharedQueryMaterialization { candidate, .. } - | Self::AllowedOutputs { candidate, .. } - | Self::IncompatibleOutputs { candidate, .. } => candidate, - } - } -} - -/// Index into [`DagOfSelectedCandidates::candidate_of_node`]: one run of a -/// candidate. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct SelectedNodeId(pub usize); - -/// One edge of a [`DagOfSelectedCandidates`]: `producer`'s output is input -/// `input` of `consumer`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct SelectedEdge { - pub producer: SelectedNodeId, - pub consumer: SelectedNodeId, - /// Index in the consumer's inputs (its candidate's - /// `producer_candidates_per_input`). - pub input: usize, -} - -/// The selected plan as a DAG with one node per run of a candidate: what -/// Stage 3 selects and the facade builds through -/// [`AssignmentRealizer::realize`](crate::stages::AssignmentRealizer::realize). -/// A candidate that runs twice (recomputed by two readers, or two shared -/// instances over different inputs) is two nodes. -#[derive(Debug, Clone, PartialEq)] -pub struct DagOfSelectedCandidates { - /// Node `i` runs candidate `candidate_of_node[i]`. - pub candidate_of_node: Vec, - /// For each input of each node, the node that produces it. - pub edges: Vec, - /// The node producing each output reference, one per output reference, - /// in [`output_references`](super::output_references) order. - pub node_of_output_reference: Vec, -} - -impl DagOfSelectedCandidates { - /// Structural check against `physical`, instead of trusting rounded - /// solver values: every node runs an existing candidate, every input of - /// every node has exactly one edge from a node of an allowed producer - /// candidate, the graph is acyclic, every node is read by some output - /// reference, each output reference comes from one of its - /// `query_output_candidates`, and every node keeps its candidate's - /// reader constraints. Costs and resource budgets are Stage 3's to - /// check. - pub fn validate(&self, physical: &PhysicalASAPMemoCandidates) -> Result<(), String> { - let query_output_candidates: Vec> = (0..physical.output_reference_count()) - .map(|output| physical.query_output_candidates(output).collect()) - .collect(); - let query_output_candidates: Vec<&[_]> = - query_output_candidates.iter().map(Vec::as_slice).collect(); - self.validate_against( - physical.candidates.len(), - |candidate| { - physical.candidates[candidate.0] - .producer_candidates_per_input - .len() - }, - |candidate, input| { - physical.candidates[candidate.0] - .producer_candidates_per_input - .get(input) - }, - &query_output_candidates, - &physical.reader_constraints, - ) - } - - /// [`Self::validate`] over a catalog given by its parts: - /// `candidate_count` candidates; `input_count(c)` inputs of candidate - /// `c`; `producers_of_input(c, i)` the candidates that may produce its - /// input `i` (`None` past its last input); `query_output_candidates[r]` - /// the candidates that may produce output reference `r`. - fn validate_against<'a>( - &self, - candidate_count: usize, - input_count: impl Fn(PhysicalCandidateId) -> usize, - producers_of_input: impl Fn(PhysicalCandidateId, usize) -> Option<&'a ProducerCandidates>, - query_output_candidates: &[&[PhysicalCandidateId]], - reader_constraints: &[ReaderConstraint], - ) -> Result<(), String> { - let invalid = - || "decoded candidate selection violates dependency or root constraints".to_string(); - let count = self.candidate_of_node.len(); - if self - .candidate_of_node - .iter() - .any(|c| c.0 >= candidate_count) - || self.node_of_output_reference.len() != query_output_candidates.len() - { - return Err(invalid()); - } - for (node, allowed) in self - .node_of_output_reference - .iter() - .zip(query_output_candidates) - { - if node.0 >= count || !allowed.contains(&self.candidate_of_node[node.0]) { - return Err(invalid()); - } - } - // Per node, per input: the node producing it. - let mut producers: Vec>> = self - .candidate_of_node - .iter() - .map(|&candidate| vec![None; input_count(candidate)]) - .collect(); - let mut consumers = vec![Vec::new(); count]; - for edge in &self.edges { - let (producer, consumer) = (edge.producer.0, edge.consumer.0); - if producer >= count || consumer >= count { - return Err(invalid()); - } - let Some(producer_candidates) = - producers_of_input(self.candidate_of_node[consumer], edge.input) - else { - return Err(invalid()); - }; - let Some(target) = producers[consumer].get_mut(edge.input) else { - return Err(invalid()); - }; - if target.is_some() || !producer_candidates.contains(&self.candidate_of_node[producer]) - { - return Err(invalid()); - } - *target = Some(producer); - consumers[producer].push(consumer); - } - if producers.iter().flatten().any(Option::is_none) { - return Err(invalid()); - } - // Kahn's algorithm checks cycles without recursion on a potentially deep DAG. - let mut outstanding: Vec<_> = producers.iter().map(Vec::len).collect(); - let mut ready: Vec<_> = (0..count).filter(|&i| outstanding[i] == 0).collect(); - let mut visited = 0; - while let Some(producer) = ready.pop() { - visited += 1; - for &consumer in &consumers[producer] { - outstanding[consumer] -= 1; - if outstanding[consumer] == 0 { - ready.push(consumer); - } - } - } - if visited != count { - return Err(invalid()); - } - // The nodes each output reference reaches. - let output_reach: Vec> = self - .node_of_output_reference - .iter() - .map(|root| { - let mut reached = HashSet::new(); - let mut pending = vec![root.0]; - while let Some(node) = pending.pop() { - if reached.insert(node) { - pending.extend(producers[node].iter().flatten().copied()); - } - } - reached - }) - .collect(); - if (0..count).any(|node| !output_reach.iter().any(|reach| reach.contains(&node))) { - return Err(invalid()); - } - let queries_reaching = |node: usize, query_outputs: &[Vec]| { - query_outputs - .iter() - .filter(|outputs| { - outputs - .iter() - .any(|&output| output_reach[output].contains(&node)) - }) - .count() - }; - let output_nodes: HashSet = self - .node_of_output_reference - .iter() - .map(|node| node.0) - .collect(); - let mut nodes_of_candidate: HashMap> = HashMap::new(); - for (node, candidate) in self.candidate_of_node.iter().enumerate() { - nodes_of_candidate.entry(*candidate).or_default().push(node); - } - for constraint in reader_constraints { - let Some(nodes) = nodes_of_candidate.get(&constraint.candidate()) else { - continue; - }; - let violated = nodes.iter().any(|&node| match constraint { - ReaderConstraint::MinimumQueryReaders { - query_outputs, - minimum, - .. - } => queries_reaching(node, query_outputs) < *minimum, - ReaderConstraint::HighestSharedQueryMaterialization { query_outputs, .. } => { - !output_nodes.contains(&node) - && !consumers[node] - .iter() - .any(|&consumer| queries_reaching(consumer, query_outputs) <= 1) - } - ReaderConstraint::AllowedOutputs { outputs, .. } => { - (0..output_reach.len()).any(|output| { - !outputs.contains(&output) && output_reach[output].contains(&node) - }) - } - ReaderConstraint::IncompatibleOutputs { left, right, .. } => { - output_reach[*left].contains(&node) && output_reach[*right].contains(&node) - } - }); - if violated { - return Err("decoded selection violates selected-reader compatibility".into()); - } - } - Ok(()) - } -} diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 1d500fc27..f3b1ec719 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -1,131 +1,86 @@ -//! Stage 2's output: every operator node of every implementation candidate, -//! each with the timings and materializations Stage 2 allows, and which -//! candidates may produce each input. Stage 3 selects among them; Stage 2 -//! prices nothing. -use std::rc::Rc; +//! Stage 2's output: each logical candidate's physical alternatives +//! $\mathcal C$, one per valid $p^{\mathrm{phys}} = (\tau, \ell)$, and +//! the physical edges $\mathcal E$ with $\alpha^{\log}$ and +//! $\alpha^{\mathrm{phys}}$ (#1200, #1201). Stage 3 selects among them; +//! Stage 2 prices nothing. -use super::graph::{PhysicalCandidateId, ReaderConstraint}; -use super::producers::ProducerCandidates; -use crate::ir::properties::ExecutionTiming; -use crate::ir::{OperatorNode, QueryRoot}; -use crate::physical::Materialization; -use crate::stages::{MemoGroupId, PrunedCandidate}; +use std::collections::HashMap; -/// One implementation candidate: index `index` in group `group`'s -/// candidates ([`Memo::candidates`](crate::stages::Memo::candidates)). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct CandidateForMemoGroupId { - pub group: MemoGroupId, - pub index: usize, -} - -/// Index into -/// [`PhysicalASAPMemoCandidates::operator_nodes_of_candidates_for_memo_groups`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct OperatorNodeOfCandidateForMemoGroupId(pub usize); +use crate::ir::QueryRoot; +use crate::stages::{ + LogicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, + PhysicalEdgeId, PrunedCandidate, +}; -/// One operator node of one or more candidates for a memo group, before -/// Stage 2 chose its timing and materialization. +/// Stage 2's output, which Stage 3 selects from. #[derive(Debug, Clone)] -pub struct OperatorNodeOfCandidateForMemoGroup { - /// Every candidate of the group this node can implement, ascending: - /// identical terminal readers of different candidates are factored into - /// one node, and the selected producers decide which one it implements. - /// A private node belongs to the one candidate it was lowered from. - pub candidates_for_memo_group: Vec, - /// Whether it is its candidates' root; false for an implementation's - /// private nodes. - pub is_implementation_root: bool, - /// How many equal shares of a query's accuracy budget this node's sketch sizes meet - /// (`CandidateForMemoGroup::accuracy_budget_parts`). For a factored node, the largest among its - /// candidates: equal nodes have equal sizes. 1 for an exact node. - pub accuracy_budget_parts: usize, - /// The operator node, with Stage 2's physical implementation; its - /// children are the expected inputs, which the selected producers replace. - pub physical_node: Rc, +pub struct PhysicalASAPMemoCandidates { + /// Pass 2's output: the memo, its candidates' $e_v$, $p^{\log}_v$, + /// $r^{\log}_{c,i}$, and the logical edges. + pub logical_candidates: LogicalASAPMemoCandidates, + /// $\mathcal C$, indexed by [`PhysicalCandidateId`]. + pub physical_candidates: Vec, + /// The alternatives Stage 2 pruned, in pruning order, each as the + /// [`PhysicalCandidate`] it would have been: candidates are numbered + /// after pruning, so a pruned one has no [`PhysicalCandidateId`]. + pub pruned_candidates: Vec>, + /// $\mathcal E$, grouped by (consumer, input); indexed by + /// [`PhysicalEdgeId`]. + edges: Vec, + /// $P_{c,i}$ after $\alpha$: the admissible edges into each (consumer, input). + admissible_edges_into_input: HashMap<(PhysicalCandidateId, usize), Vec>, + /// The admissible edges out of each candidate, by [`PhysicalCandidateId`]. + admissible_edges_out_of_candidate: Vec>, } -impl OperatorNodeOfCandidateForMemoGroup { - /// The memo group whose candidates this node implements. - /// `candidates_for_memo_group` is never empty, and all its entries are - /// candidates of this one group. - pub fn group(&self) -> MemoGroupId { - let group = self.candidates_for_memo_group[0].group; - debug_assert!(self - .candidates_for_memo_group - .iter() - .all(|candidate| candidate.group == group)); - group +impl PhysicalASAPMemoCandidates { + /// Stage 2's output over `physical_candidates` and `edges` (grouped by + /// (consumer, input)), with the admissible-edge indexes built once. + /// Query roots are read from `logical_candidates`, not copied. + pub fn new( + logical_candidates: LogicalASAPMemoCandidates, + physical_candidates: Vec, + edges: Vec, + pruned_candidates: Vec>, + ) -> Self { + let _ = ( + logical_candidates, + physical_candidates, + edges, + pruned_candidates, + ); + todo!("#1207 PR 5: index the admissible edges") } -} -/// One operator node with one timing and materialization: what Stage 3 -/// selects. A selection may run it more than once, each run a node of the -/// [`DagOfSelectedCandidates`](super::DagOfSelectedCandidates). -#[derive(Debug, Clone)] -pub struct PhysicalCandidate { - /// The operator node it runs. - pub operator_node: OperatorNodeOfCandidateForMemoGroupId, - /// When it runs. A node that is not materialized under an ingestion-time - /// build still runs at ingestion time, so both are kept. - pub timing: ExecutionTiming, - /// Whether and where its result is kept. - pub materialization: Materialization, - /// Per input of its operator node (in `map_children` order), the - /// candidates that may produce it. Depends on its timing. - pub producer_candidates_per_input: Vec, - /// Whether it may produce a query's output: an implementation root at - /// query time whose output matches its group's logical node, in a group - /// some query's root reads. - pub may_produce_query_output: bool, - /// One entry per workload query: whether this tumbling-window chain may be kept - /// across evaluations when that query is its only reader. Filled only - /// for a `MatAtQueryKeptAcrossEvaluations(Memory)` tumbling-window chain in a - /// deployment that keeps query-time results; empty otherwise. - pub may_keep_across_evaluations_per_sole_reader_query: Vec, -} + /// $\mathcal E$: every edge, in (consumer, input) order. + pub fn edges(&self) -> impl Iterator + '_ { + self.edges + .iter() + .enumerate() + .map(|(id, edge)| (PhysicalEdgeId(id), *edge)) + } -/// A physical candidate Stage 2 pruned, named by what it would have been: -/// candidates are renumbered after pruning, so a pruned one has no -/// [`PhysicalCandidateId`]. Unique among a [`PhysicalASAPMemoCandidates`]' -/// `pruned_candidates`: an operator node has at most one candidate per -/// timing and materialization. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct PhysicalCandidateKey { - /// Its operator node: an index into the output's - /// `operator_nodes_of_candidates_for_memo_groups`, which pruning keeps - /// whole. - pub operator_node: OperatorNodeOfCandidateForMemoGroupId, - pub timing: ExecutionTiming, - pub materialization: Materialization, -} + /// $P^{\mathrm{adm}}_{c,i}$: the admissible edges into input + /// `input_index_of_consumer` of `consumer` + /// ($\alpha^{\log} = \alpha^{\mathrm{phys}} = 1$). + pub fn admissible_edges_into( + &self, + consumer: PhysicalCandidateId, + input_index_of_consumer: usize, + ) -> &[PhysicalEdgeId] { + let _ = ( + consumer, + input_index_of_consumer, + &self.admissible_edges_into_input, + ); + todo!("#1207 PR 5") + } -/// Stage 2's output, which Stage 3 selects from. -#[derive(Debug, Clone)] -pub struct PhysicalASAPMemoCandidates { - /// One per operator node of an implementation, before Stage 2 chose its - /// timing and materialization. - pub operator_nodes_of_candidates_for_memo_groups: Vec, - /// One per (operator node, timing, materialization): what Stage 3 - /// selects. Indexed by [`PhysicalCandidateId`]. - pub candidates: Vec, - /// Each query's root over groups, by index in the workload, passed - /// through from Pass 2. Its group references, in order, are the output - /// references of [`Self::query_output_candidates`]. - pub query_roots: Vec>, - /// Legality that depends on which readers are selected. - pub reader_constraints: Vec, - /// The candidates Stage 2 pruned, each on its operator node's memo - /// group, in pruning order: configurations it does not generate, then - /// candidates no buildable input, query output or reader count can use. - /// Empty when built by [`Self::new`]. - pub pruned_candidates: Vec>, - /// By group index: the candidates whose operator node is an - /// implementation root of that group, ascending. - implementation_roots_of_group: Vec>, - /// Each output reference's group: [`output_references`] of - /// `query_roots`, built once. - output_reference_groups: Vec, + /// The admissible edges out of `producer`: every consumer that may share it. + pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { + let _ = (producer, &self.admissible_edges_out_of_candidate); + todo!("#1207 PR 5") + } } /// The output references of `query_roots`: each operator reference, in @@ -145,82 +100,3 @@ pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, } references } - -impl PhysicalASAPMemoCandidates { - /// Builds the private per-group index over `candidates`. Stage 3's - /// component search uses it to build one smaller catalog per component. - pub fn new( - operator_nodes_of_candidates_for_memo_groups: Vec, - candidates: Vec, - query_roots: Vec>, - reader_constraints: Vec, - ) -> Self { - let mut implementation_roots_of_group: Vec> = Vec::new(); - for (id, candidate) in candidates.iter().enumerate() { - let node = &operator_nodes_of_candidates_for_memo_groups[candidate.operator_node.0]; - if node.is_implementation_root { - let group = node.group().0 as usize; - if implementation_roots_of_group.len() <= group { - implementation_roots_of_group.resize(group + 1, Vec::new()); - } - implementation_roots_of_group[group].push(PhysicalCandidateId(id)); - } - } - let output_reference_groups = output_references(&query_roots) - .into_iter() - .map(|(_, group)| group) - .collect(); - Self { - operator_nodes_of_candidates_for_memo_groups, - candidates, - query_roots, - reader_constraints, - pruned_candidates: Vec::new(), - implementation_roots_of_group, - output_reference_groups, - } - } - - /// The implementation roots of `group`, ascending. - pub fn implementation_roots(&self, group: MemoGroupId) -> &[PhysicalCandidateId] { - self.implementation_roots_of_group - .get(group.0 as usize) - .map_or(&[], Vec::as_slice) - } - - /// The candidates that may produce output reference `output_reference` - /// (each group reference of `query_roots`, in order; a scalar query has - /// several): its group's roots with `may_produce_query_output`. - pub fn query_output_candidates( - &self, - output_reference: usize, - ) -> impl Iterator + '_ { - let roots = self - .output_reference_groups - .get(output_reference) - .map_or(&[][..], |&group| self.implementation_roots(group)); - roots - .iter() - .copied() - .filter(|id| self.candidates[id.0].may_produce_query_output) - } - - /// The number of output references: group references of `query_roots`. - pub fn output_reference_count(&self) -> usize { - self.output_reference_groups.len() - } - - /// The operator node candidate `candidate` runs. - pub fn physical_node(&self, candidate: PhysicalCandidateId) -> &Rc { - &self.operator_nodes_of_candidates_for_memo_groups - [self.candidates[candidate.0].operator_node.0] - .physical_node - } - - /// The memo group whose candidates candidate `candidate` implements. - pub fn memo_group_of(&self, candidate: PhysicalCandidateId) -> MemoGroupId { - self.operator_nodes_of_candidates_for_memo_groups - [self.candidates[candidate.0].operator_node.0] - .group() - } -} diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index 7f029925a..b733ee832 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -1,18 +1,17 @@ -//! The Stage 2 → Stage 3 boundary: the physical candidates Stage 2 builds -//! from the memo and Stage 3 selects from (#1047, #1054). +//! The Stage 2 → Stage 3 boundary: the physical candidates and edges Stage 2 +//! builds from the memo and Stage 3 selects from (#1047, #1054, #1200, #1201), +//! by scope (execution context → candidate → edge → Stage 2's output). -mod graph; +mod candidate; +mod edge; +mod execution_context; mod memo_candidates; -mod producers; +mod properties; -pub use graph::{ - DagOfSelectedCandidates, PhysicalCandidateId, ReaderConstraint, SelectedEdge, SelectedNodeId, +pub use candidate::{ + producer_satisfies_physical_requirements, PhysicalCandidate, PhysicalCandidateId, }; -pub use memo_candidates::{ - output_references, CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, - OperatorNodeOfCandidateForMemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, - PhysicalCandidateKey, -}; -#[cfg(any(test, feature = "test-support"))] -pub use producers::{producer_storage_metrics, ProducerStorageMetrics}; -pub use producers::{ProducerCandidates, ProducerSetInterner}; +pub use edge::{PhysicalEdge, PhysicalEdgeId}; +pub use execution_context::{EvaluationGrid, ExecutionContext, ExecutionContextId}; +pub use memo_candidates::{output_references, PhysicalASAPMemoCandidates}; +pub use properties::{Lifetime, PhysicalProperties, PhysicalPropertiesError, StorageMedium}; diff --git a/crates/types/src/stages/physical/producers.rs b/crates/types/src/stages/physical/producers.rs deleted file mode 100644 index 0e7547434..000000000 --- a/crates/types/src/stages/physical/producers.rs +++ /dev/null @@ -1,206 +0,0 @@ -//! Shared storage for ordered compatible producers. Sharing this storage never -//! ties together the independent selections of different consumer input slots. -use super::graph::PhysicalCandidateId; -use std::collections::HashSet; -use std::ops::{Deref, DerefMut}; -use std::rc::Rc; - -#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] -pub struct ProducerCandidates(Rc>); - -impl From> for ProducerCandidates { - fn from(values: Vec) -> Self { - Self(Rc::new(values)) - } -} - -impl FromIterator for ProducerCandidates { - fn from_iter>(values: T) -> Self { - Self::from(values.into_iter().collect::>()) - } -} - -impl Deref for ProducerCandidates { - type Target = Vec; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for ProducerCandidates { - fn deref_mut(&mut self) -> &mut Self::Target { - Rc::make_mut(&mut self.0) - } -} - -impl<'a> IntoIterator for &'a ProducerCandidates { - type Item = &'a PhysicalCandidateId; - type IntoIter = std::slice::Iter<'a, PhysicalCandidateId>; - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> IntoIterator for &'a mut ProducerCandidates { - type Item = &'a mut PhysicalCandidateId; - type IntoIter = std::slice::IterMut<'a, PhysicalCandidateId>; - fn into_iter(self) -> Self::IntoIter { - self.iter_mut() - } -} - -impl IntoIterator for ProducerCandidates { - type Item = PhysicalCandidateId; - type IntoIter = std::vec::IntoIter; - fn into_iter(self) -> Self::IntoIter { - Rc::unwrap_or_clone(self.0).into_iter() - } -} - -/// Exact ordered-content interning. No sorting, deduplication, or compatibility -/// inference occurs here; malformed input remains visible to graph validation. -#[derive(Default)] -pub struct ProducerSetInterner { - sets: HashSet, -} - -impl ProducerSetInterner { - pub fn intern(&mut self, values: impl Into) -> ProducerCandidates { - let values = values.into(); - if let Some(existing) = self.sets.get(&values) { - return existing.clone(); - } - self.sets.insert(values.clone()); - values - } -} - -/// Count physical producer-list storage separately from logical connections. -/// Distinct input positions remain distinct even when their storage is shared. -#[cfg(any(test, feature = "test-support"))] -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct ProducerStorageMetrics { - pub logical_connections: usize, - pub unique_producer_sets: usize, - pub stored_producer_ids: usize, -} - -#[cfg(any(test, feature = "test-support"))] -pub fn producer_storage_metrics( - physical: &super::PhysicalASAPMemoCandidates, -) -> ProducerStorageMetrics { - let mut allocations = HashSet::new(); - let mut metrics = ProducerStorageMetrics::default(); - for producer_candidates in physical - .candidates - .iter() - .flat_map(|candidate| &candidate.producer_candidates_per_input) - { - metrics.logical_connections += producer_candidates.len(); - if allocations.insert(Rc::as_ptr(&producer_candidates.0)) { - metrics.unique_producer_sets += 1; - metrics.stored_producer_ids += producer_candidates.len(); - } - } - metrics -} - -#[cfg(test)] -mod tests { - use super::*; - - // Two slots can choose different producers even when their compatible lists - // share one allocation; storage compression adds no selection equality. - #[test] - fn shared_producer_storage_keeps_input_choices_independent() { - use super::super::graph::{DagOfSelectedCandidates, SelectedEdge, SelectedNodeId}; - use super::super::memo_candidates::{ - CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, - OperatorNodeOfCandidateForMemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, - }; - use crate::ir::properties::ExecutionTiming; - use crate::ir::QueryRoot; - use crate::physical::Materialization; - use crate::stages::MemoGroupId; - let mut interner = ProducerSetInterner::default(); - let choices = interner.intern(vec![PhysicalCandidateId(0), PhysicalCandidateId(1)]); - use crate::ir::scalar::expr_ir::ScalarValue; - use crate::ir::schema::{DataType, Field, Schema}; - use crate::ir::{NonASAPOp, Operator, OperatorNode, ScalarExpr}; - let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(1.0))]], - schema: Schema::lifted(vec![Field::plain("value", DataType::Float64, false)], None), - })) - .unwrap(); - let physical = PhysicalASAPMemoCandidates::new( - (0..3) - .map(|group| OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![CandidateForMemoGroupId { - group: MemoGroupId(group), - index: 0, - }], - is_implementation_root: true, - accuracy_budget_parts: 1, - physical_node: Rc::clone(&node), - }) - .collect(), - (0..3) - .map(|id| PhysicalCandidate { - operator_node: OperatorNodeOfCandidateForMemoGroupId(id), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - producer_candidates_per_input: if id == 2 { - vec![choices.clone(), choices.clone()] - } else { - vec![] - }, - may_produce_query_output: id == 2, - may_keep_across_evaluations_per_sole_reader_query: Vec::new(), - }) - .collect(), - vec![QueryRoot::Operator(MemoGroupId(2))], - Vec::new(), - ); - let selected = DagOfSelectedCandidates { - candidate_of_node: (0..3).map(PhysicalCandidateId).collect(), - edges: (0..2) - .map(|input| SelectedEdge { - producer: SelectedNodeId(input), - consumer: SelectedNodeId(2), - input, - }) - .collect(), - node_of_output_reference: vec![SelectedNodeId(2)], - }; - selected.validate(&physical).unwrap(); - assert_eq!( - producer_storage_metrics(&physical), - ProducerStorageMetrics { - logical_connections: 4, - unique_producer_sets: 1, - stored_producer_ids: 2 - } - ); - } - - // Equal ordered lists share one allocation; mutation of one input detaches - // it and cannot silently change another input or an interner key. - #[test] - fn interned_inputs_detach_on_mutation() { - let mut interner = ProducerSetInterner::default(); - let original = interner.intern(vec![PhysicalCandidateId(2), PhysicalCandidateId(1)]); - let mut changed = interner.intern(vec![PhysicalCandidateId(2), PhysicalCandidateId(1)]); - assert!(Rc::ptr_eq(&original.0, &changed.0)); - changed.retain(|id| id.0 == 1); - assert_eq!( - original.as_slice(), - &[PhysicalCandidateId(2), PhysicalCandidateId(1)] - ); - assert_eq!(changed.as_slice(), &[PhysicalCandidateId(1)]); - assert!(!Rc::ptr_eq(&original.0, &changed.0)); - let repeated = interner.intern(original.clone()); - assert!(Rc::ptr_eq(&original.0, &repeated.0)); - let reversed = interner.intern(vec![PhysicalCandidateId(1), PhysicalCandidateId(2)]); - assert!(!Rc::ptr_eq(&original.0, &reversed.0)); - } -} diff --git a/crates/types/src/stages/physical/properties.rs b/crates/types/src/stages/physical/properties.rs new file mode 100644 index 000000000..df1869ed4 --- /dev/null +++ b/crates/types/src/stages/physical/properties.rs @@ -0,0 +1,120 @@ +//! A physical candidate's physical properties $p^{\mathrm{phys}}_v = (\tau, \ell)$ +//! (#1200; concepts doc `memo-groups-properties-requirements.md`, sections 1 +//! and 3): when it runs ([`ExecutionTiming`]) and how long its output lives +//! ([`Lifetime`]). There is no schedule property: a `None` or `OneEvaluation` +//! output runs on the schedule of the context κ that runs it, and a tumbling +//! grid is anchored at phase 0 with its width in the logical window layout. +//! +//! | timing | lifetime | runs | +//! |---|---|---| +//! | query | `None` | in each context κ that reads it | +//! | query | `OneEvaluation` | in one context κ; that evaluation's consumers share it | +//! | query | `AcrossEvaluations` | one window when each window ends | +//! | ingestion | `None` | inside the ingestion work that reads it | +//! | ingestion | `AcrossEvaluations` | one window when each window ends | +//! | ingestion | `OneEvaluation` | invalid: there is no evaluation at ingestion time | +//! +//! There is no stored physical requirement: $r^{\mathrm{phys}}_{c,i}$ is a +//! function of the consumer's own properties +//! ([`crate::stages::producer_satisfies_physical_requirements`]). + +use crate::ir::properties::ExecutionTiming; + +/// Where a materialized output is stored. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum StorageMedium { + Memory, + Disk, +} + +/// $\ell_v$: how long a candidate's output lives. Only a materialized output +/// has a medium. +#[derive( + Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum Lifetime { + /// Discarded after its consumer's execution completes. + #[default] + None, + /// Materialized until every consumer within one evaluation finishes. + OneEvaluation(StorageMedium), + /// Materialized across evaluations, so later evaluations reuse it. + AcrossEvaluations(StorageMedium), +} + +/// A timing and lifetime that cannot go together (the invalid combination +/// of the concepts doc, section 2). +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum PhysicalPropertiesError { + #[error("an ingestion-time output cannot be materialized for one evaluation: there is no evaluation at ingestion time")] + OneEvaluationAtIngestion, +} + +/// $p^{\mathrm{phys}}_v = (\tau, \ell)$. [`Self::new`] rejects the invalid +/// combination (concepts doc, section 2). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] +pub struct PhysicalProperties { + timing: ExecutionTiming, + lifetime: Lifetime, +} + +impl PhysicalProperties { + /// The properties `(timing, lifetime)`, unless they cannot go together. + pub fn new( + timing: ExecutionTiming, + lifetime: Lifetime, + ) -> Result { + let _ = (timing, lifetime); + todo!("#1207 PR 1: reject the invalid combination of section 2") + } + + /// $\tau_v$. + pub fn timing(&self) -> ExecutionTiming { + self.timing + } + + /// $\ell_v$. + pub fn lifetime(&self) -> Lifetime { + self.lifetime + } + + /// Whether $\ell_v \ne$ None. Derived, not stored. + pub fn is_materialized(&self) -> bool { + self.lifetime != Lifetime::None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ExecutionTiming::{IngestionTime, QueryTime}; + + const MEMORY: StorageMedium = StorageMedium::Memory; + + /// The one invalid combination of the concepts doc, section 2. + #[test] + fn new_rejects_one_evaluation_at_ingestion() { + assert_eq!( + PhysicalProperties::new(IngestionTime, Lifetime::OneEvaluation(MEMORY)), + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) + ); + } + + /// The valid (τ, ℓ) pairs of section 2. + #[test] + fn new_accepts_each_valid_pair() { + for (timing, lifetime) in [ + (QueryTime, Lifetime::None), + (IngestionTime, Lifetime::None), + (QueryTime, Lifetime::OneEvaluation(MEMORY)), + (QueryTime, Lifetime::AcrossEvaluations(MEMORY)), + ( + IngestionTime, + Lifetime::AcrossEvaluations(StorageMedium::Disk), + ), + ] { + let properties = PhysicalProperties::new(timing, lifetime).unwrap(); + assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + } + } +} diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index f857460f4..9d28aa324 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -1,96 +1,96 @@ //! What a stage removed, and why: one shape for every stage (#1047 P6). //! Each stage removes only what its own knowledge rules out (option C): -//! Pass 1 what the measure or the deployment cannot compute, Stage 2 what -//! cannot be implemented, built or kept. Stage 0 fails with an error and -//! Pass 2 only adds candidates, so neither prunes. Stage 3 prunes before +//! Stage 2 what cannot be built, run or read by a query output. Stage 0 fails with an +//! error, Pass 1 records nothing it leaves out (#1205 will show it), and +//! Pass 2 records its eliminated candidates with their own causes +//! (`PrunedLogicalCandidate`). Stage 3 prunes before //! its selection MILP (dominated and uncostable candidates); its //! rejections of whole plans keep their own cause (`RejectionCause` in plan -//! selection). +//! selection). The planner does not check whether an executor implements an +//! operator or summary type, so no reason names it. -use super::{MemoGroupId, PhysicalCandidateId}; +use crate::accuracy::AccuracyMetric; +use crate::stages::{CandidateForMemoGroupId, PhysicalCandidateId, RequiredAccuracy}; -/// One candidate a stage removed from memo group `group`, and why. `C` is -/// the stage's candidate type: Pass 1's way to compute a measure -/// (`HowToComputeAggregateMeasure`), Stage 2's -/// [`PhysicalCandidateKey`](super::PhysicalCandidateKey). +/// A candidate eliminated by Pass 2. +#[derive(Debug, Clone)] +pub struct PrunedLogicalCandidate { + pub candidate: CandidateForMemoGroupId, + /// Every cause: one per failing edge it is cut off by, so mixed causes + /// (a missed requirement on one edge, the metric rule on another) are + /// all listed. + pub causes: Vec, +} + +/// One path from the candidate, through consumers that are themselves +/// eliminated, to the edge that fails. One shortest path per failing edge. +#[derive(Debug, Clone)] +pub struct PruneCause { + /// The eliminated consumers on the path, nearest first. + pub through_pruned_consumers: Vec, + pub root: RootPruneReason, +} + +/// Why the edge at the end of a [`PruneCause`]'s path fails, or why the +/// candidate has no edge at all. +#[derive(Debug, Clone, serde::Serialize)] +pub enum RootPruneReason { + /// The producer's guarantee, at the required failure probability, + /// exceeds the consumer input's required error. + BelowRequiredAccuracy { + consumer: CandidateForMemoGroupId, + input: usize, + required: RequiredAccuracy, + }, + /// The consumer's error contract carries none of the producer's metrics. + MetricRule { + consumer: CandidateForMemoGroupId, + input: usize, + producer_metric: AccuracyMetric, + consumer_carries: Vec, + }, + /// The candidate never had a consumer and is not a query output. A + /// candidate whose consumers were eliminated is traced to their causes. + NoConsumer, +} + +/// One candidate a stage removed, and why: Stage 2 and Stage 3 record the +/// [`PhysicalCandidate`](super::PhysicalCandidate). The candidate names its +/// group through its logical candidate, so there is no separate group field. #[derive(Debug, Clone, PartialEq)] pub struct PrunedCandidate { - pub group: MemoGroupId, pub candidate: C, pub reason: PruneReason, } -/// Why a stage removed a candidate. The first seven are Pass 1's: the -/// candidate cannot compute its measure, the deployment cannot run it, or -/// no query that reads it can meet its accuracy target with it. -/// Then Stage 2's: the physical candidate cannot be built, used or -/// kept. The last two are Stage 3's, before its selection MILP. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +/// Why a stage removed a candidate: first Stage 2's (the physical candidate +/// cannot be built, run or read by a query output), then Stage 3's, before +/// its selection MILP. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] pub enum PruneReason { - /// Pass 1: the measure requires an exact answer and the summary is - /// approximate. - RequiresExact, - /// Pass 1: `count` counts rows, so a possibly-null `avg` input changes - /// the result. - NullableAvgInput, - /// Pass 1: frequency sketches do not implement SQL row-count updates. - /// The exact counter is kept; a fused raw-row top-k has its own keyed - /// update contract. - NoRelationalCountSketchRealization, - /// Pass 1: the deployment's executor cannot build the summary's family - /// or answer the measure's readout from it (`summary_supported` in - /// Stage 1). - UnsupportedByDeployment, - /// Pass 1: the quantile's φ is outside [0, 1], where PromQL answers -Inf - /// or +Inf: no sketch estimates that, and the executor's sketches refuse - /// such a φ, so only the exact answer remains. - QuantileOutsideUnitInterval, - /// Pass 1: a PromQL top-k's heap (Count-Min or CountSketch with a heap) - /// over its own input, an instant vector with one row per ranked series - /// or group: the heap reads every row and adds nothing to it, so an - /// exact top-k over the same rows, which keeps only `k` of them, is - /// exact and no costlier. Only the whole-expression heap over the inner - /// aggregate's input, where an item's rows repeat, is offered (Pass 1's - /// fused computation). - TopKHeapOverOneRowPerItem, - /// Pass 1: for every query that reads its memo group, the query's - /// output has no error bound that can meet the query's accuracy target - /// at any summary size: guarantee propagation has no rule above it - /// (e.g. a rank error through `avg`, or a value `HAVING` above an - /// estimate), its bound stays unknown once the data workload's - /// declarations are applied (e.g. a relative error through `avg` over - /// samples not declared non-negative), its bound is in a metric the - /// target does not accept (the deployment's accuracy model decides), or - /// its failure probability is per value and the target's is over a - /// whole result whose number of values the data does not bound. Kept - /// when another aggregate over the same source offers the same family, - /// which Pass 2 may build from it. Pruned only when the deployment's - /// accuracy model and evidence cannot accept an output without its - /// guarantee. - NoErrorBoundForQueryTarget, - /// Stage 2: the executor cannot run its operator node, an exact - /// per-entity `Aggregate` (a per-series range reduction over raw data - /// samples) with more than one measure, a measure without an exact - /// per-entity kernel (e.g. `stddev_over_time`), a measure filter or a - /// `HAVING` (`agg_has_exact_per_entity_kernel`). Every timing and - /// materialization of the node is pruned. - PerEntityAggregateNotExecutable, + /// Stage 2: it runs at ingestion time, and the source has no ingestion + /// pipeline (its data is at rest or its arrival unknown). + NoIngestionPipeline, /// Stage 2: it scans raw data at query time, and the deployment does /// not retain raw data, so no window of it is there to scan. RawDataNotRetained, + /// Stage 2: it is a query-time result materialized (`OneEvaluation` or + /// `AcrossEvaluations`), and the deployment retains no query-time + /// results. + QueryTimeResultsNotRetained, /// Stage 2: one of its inputs has no producer candidate that can be /// built: none is left, or every one depends, directly or through a /// cycle, on an input that cannot be built. InputHasNoBuildableProducer, /// Stage 2: no query output can read it, directly or through other - /// candidates, once each candidate's allowed readers are applied. + /// candidates, once each candidate's allowed consumers are applied. ReachesNoQueryOutput, - /// Stage 2: it is shared within one query evaluation - /// (`MatAtQuery`), which needs at least two reading queries, and fewer - /// queries can reach it. - TooFewQueryReaders, + /// Stage 2: a Count-Min build (`Cms`, `CmsWithHeap`, `HydraCms`) whose + /// weight column's `column_sign` is not `NonNegative`: its estimates are + /// bounds only for non-negative weights. + CountMinWeightsNotProvenNonNegative, /// Stage 3: the candidate it names, in the same memo group with the - /// same producers and readers, is at least as good in every respect + /// same producers and consumers, is at least as good in every respect /// (guarantee, usage, latency, output, held state), so no optimal plan /// needs this one. DominatedBy(PhysicalCandidateId), diff --git a/crates/types/src/stages/realizer.rs b/crates/types/src/stages/realizer.rs deleted file mode 100644 index 8c2ff17d6..000000000 --- a/crates/types/src/stages/realizer.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! The Stage 3 → Stage 2 callback (#1054 X1): Stage 3 proposes a DAG of -//! selected candidates, and Stage 2 realizes it. Stage 3 receives a `&dyn AssignmentRealizer` from the facade, -//! so it never depends on the Stage 2 crate. -use std::collections::BTreeMap; -use std::rc::Rc; - -use crate::ir::physical_export::PhysicalASAPNodeId; -use crate::ir::OperatorNode; -use crate::physical::PhysicalWorkloadPlan; - -use super::{DagOfSelectedCandidates, PhysicalASAPMemoCandidates}; - -/// Why a selected assignment could not be realized. Only rejected -/// assignments are safe to exclude from the finite search; adapter or -/// identity failures must stop with diagnostics, not become no-good cuts. -#[derive(Debug, thiserror::Error)] -pub enum PhysicalCandidateError { - #[error("selected physical assignment is infeasible: {0}")] - RejectedAssignment(String), - #[error("invalid or unsupported physical reconstruction: {0}")] - InvalidOrUnsupported(String), -} - -impl From for PhysicalCandidateError { - fn from(message: String) -> Self { - Self::InvalidOrUnsupported(message) - } -} -impl From<&str> for PhysicalCandidateError { - fn from(message: &str) -> Self { - Self::InvalidOrUnsupported(message.into()) - } -} - -/// A realized selection (was `SelectedPhysicalCandidate`): the physical -/// plan Stage 2 built for it, how long each kept tumbling-window chain of that plan -/// keeps its tumbling windows, and the operator node each selected node became. -#[derive(Debug, Clone)] -pub struct RealizedPlan { - /// The selection rebuilt as one physical plan, one operator node per - /// selected node, each with its candidate's timing and materialization. - pub plan: PhysicalWorkloadPlan, - /// For each tumbling-window chain of `plan` kept beyond one evaluation - /// (maintained at ingestion time, or built at query time and kept - /// across evaluations), by node id in `plan.dag`: how far back from the - /// newest evaluation the runtime must keep its completed tumbling windows, in - /// milliseconds. It is how far its tumbling windows reach back: the offset its - /// newest tumbling window ends before the evaluation (`[60m] offset 10m`: 10 min), - /// plus the chain's window, plus the longest range-evaluation span of - /// its readers, so the earliest step of the next request still finds - /// its tumbling windows; without an offset or range evaluations, just the window. - /// Empty when the plan keeps no tumbling windows. - pub tumbling_window_retention_ms: BTreeMap, - /// The operator node of `plan` each selected node became, indexed by - /// [`SelectedNodeId`](super::SelectedNodeId). - pub operator_node_of_selected_node: Vec>, -} - -/// Realizes a [`DagOfSelectedCandidates`] through Stage 2, one plan node -/// per selected node. Implemented by the -/// Stage 2 crate, which holds the planning context (the workload's -/// original query roots, its entries and data, and whether query-time -/// results are retained); the facade builds it, passes it to Stage 3, and -/// calls the same method to build the selected plan. -/// -/// Guarantees: a returned result has one operator node per selected node, each -/// rebuilt over its selected producers with its identity kept, guarantees -/// recomposed, and every output contract (result kind and schema) met; a -/// realized assignment also passes Stage 2's timing and dependency rules -/// and the physical DAG validator, with every node keeping its -/// candidate's timing and materialization. -/// -/// It checks structure and execution legality only. It does not check cost, -/// and it does not check accuracy targets: Stage 3 does both. -pub trait AssignmentRealizer { - fn realize( - &self, - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, - ) -> Result; -} diff --git a/crates/types/src/ir/schema/family_capability.rs b/crates/types/src/summary_type/capability.rs similarity index 53% rename from crates/types/src/ir/schema/family_capability.rs rename to crates/types/src/summary_type/capability.rs index 6af62d919..612f854ae 100644 --- a/crates/types/src/ir/schema/family_capability.rs +++ b/crates/types/src/summary_type/capability.rs @@ -1,31 +1,31 @@ -//! The generic family-capability table: what each summary family can do, -//! keyed by its [`FieldDataType`]. +//! The summary-type capability table: what each summary type can do, keyed +//! by its [`FieldDataType`] (family and algorithm). //! -//! The IR stays permissive: a `SummaryMerge` over any state family is +//! The IR stays permissive: a `SummaryMerge` over any state type is //! structurally valid. The planner rules and the executor consult this //! table to decide whether a composition is sound. It is the one source of -//! which statistic a family answers: the IR's `SummaryEstimate` validation, -//! the executor's evaluation check and its supported-summary list all read -//! it. One row per family says +//! which statistic a summary type answers: the IR's `SummaryEstimate` +//! validation, the executor's evaluation check and its supported-summary +//! list all read it. One row per summary type says //! how the inputs of a merge must relate ([`MergeInputRule`]) and which //! statistics ([`StatisticKind`]) one state answers, each with the error of //! that readout from a merged state ([`MergedError`]), so one state can //! serve every consumer whose statistic it answers. //! -//! A sketch family's row is `asap_sketchlib`'s -//! ([`SketchAlgorithm::capability`]), read unchanged: the family, +//! A sketch algorithm's row is `asap_sketchlib`'s +//! ([`SketchAlgorithm::capability`]), read unchanged: the algorithm, //! statistic and merged-error vocabulary is the library's. The planner //! writes the rows of the exact accumulators, which the library does not //! have. Which readouts an executor evaluates is its own declaration //! (`DeploymentCapabilities::supported_summaries`), not this table's. -use super::sketch::{ExactKind, GroupingStrategy, HydraKind, SketchAlgorithm, SketchStatistic}; +use super::{ExactKind, SketchAlgorithm, SketchStatistic}; use crate::ir::schema::FieldDataType; pub use asap_sketchlib::{MergedError, StatisticKind}; -/// What one summary family can do: its row of the family-capability table. +/// What one summary type can do: its row of the capability table. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct FamilyCapability { +pub struct SummaryTypeCapability { /// How the inputs of a merge of the family's states must relate; `None` /// when the family has no sound merge. pub merge_inputs: Option, @@ -37,15 +37,15 @@ pub struct FamilyCapability { pub(crate) readouts: &'static [(StatisticKind, MergedError)], } -/// One sketch family a deployment's executor can build, with the readouts -/// it can answer from the family's state: an entry of +/// One sketch summary type a deployment's executor can build, with the +/// readouts it can answer from its state: an entry of /// `DeploymentCapabilities::supported_summaries`. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] pub struct SupportedSummary { - /// The family the executor can build. - pub family: SketchAlgorithm, - /// The statistics it can answer from one state of the family. + /// The summary type the executor can build. + pub summary_type: SketchAlgorithm, + /// The statistics it can answer from one state of the summary type. pub readouts: Vec, } @@ -89,7 +89,7 @@ impl From for MergeInputRule { } #[cfg(any(test, feature = "test-support"))] -impl FamilyCapability { +impl SummaryTypeCapability { /// Each statistic one state of the family answers, with the error of /// that readout from a merged state. pub fn readouts(&self) -> &'static [(StatisticKind, MergedError)] { @@ -97,20 +97,28 @@ impl FamilyCapability { } } -impl FamilyCapability { - /// Whether a rollup may merge different groups' states of `family`: its - /// merge takes disjoint or overlapping inputs, and each group has its own - /// state. Counter states merge consecutive windows of one series; a - /// family without a merge merges nothing; a Hydra's groups share one - /// state. - pub fn merges_groups(family: &FieldDataType) -> bool { +impl SummaryTypeCapability { + /// Whether a rollup may merge different groups' states of + /// `summary_type`: its merge takes disjoint or overlapping inputs, and + /// each group has its own state. Counter states merge consecutive + /// windows of one series; a summary type without a merge merges nothing; + /// a Hydra's groups share one state. + pub fn merges_groups(summary_type: &FieldDataType) -> bool { matches!( - Self::of(family).merge_inputs, + Self::of(summary_type).merge_inputs, Some(MergeInputRule::Disjoint | MergeInputRule::MayOverlap) - ) && !matches!( - family, - FieldDataType::Sketch(_, grouping) - if *grouping != GroupingStrategy::PerSubpopulationInstance + ) && !Self::shares_one_state_across_groups(summary_type) + } + + /// Whether `summary_type` is a Hydra algorithm: one structure shared by + /// every group, not one state per group. + pub fn shares_one_state_across_groups(summary_type: &FieldDataType) -> bool { + use SketchAlgorithm as A; + matches!( + summary_type, + FieldDataType::Sketch( + A::HydraCms | A::HydraCountSketch | A::HydraKll | A::HydraHll | A::HydraUnivMon + ) ) } @@ -135,54 +143,38 @@ impl FamilyCapability { .map(|&(_, error)| error) } - /// The capability of a per-subpopulation state of `algorithm`, whatever - /// its parameters: `asap_sketchlib`'s row of the family, which the IR and - /// a deployment's executor read. - pub fn of_per_instance_sketch(algorithm: &SketchAlgorithm) -> Self { - let row = algorithm.capability(); - Self { - merge_inputs: row.merge_inputs.map(MergeInputRule::from), - readouts: row.readouts, - } - } - - /// The capability of `family`. Families not listed have no capability - /// (they fail closed). - pub fn of(family: &FieldDataType) -> Self { + /// The capability of `summary_type`. Summary types not listed have no + /// capability (they fail closed). + pub fn of(summary_type: &FieldDataType) -> Self { use ExactKind as E; use MergeInputRule::{Disjoint, DisjointInTime, MayOverlap}; - let (merge_inputs, readouts): (_, &'static [(StatisticKind, MergedError)]) = match family { - FieldDataType::ExactAggregate(kind, _) => match kind { - E::Sum | E::Count => (Some(Disjoint), &[]), - E::Min | E::Max => (Some(MayOverlap), &[]), - // The counter accumulator joins time-ordered windows with - // reset handling (`IncreaseAccumulator::merge_pair`). - E::Rate | E::Increase => (Some(DisjointInTime), &[]), - // IRate reads the last two samples of the window, which a - // tumbling window does not keep across its boundary. - E::IRate => (None, &[]), - }, - // A Hydra answers per subpopulation from a shared grid of the - // inner family's cells. HydraKll merges as sketchlib's row says: - // `Hydra::merge` requires the same k and m and otherwise returns - // an error (asap_sketchlib #251). Nothing builds one yet: Pass 1 - // offers no Hydra, and the executor builds only HydraCms - // (`create_hydra_cms`). - FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { - return Self::of_per_instance_sketch(&match kind { - HydraKind::HydraCms => SketchAlgorithm::HydraCms, - HydraKind::HydraCountSketch => SketchAlgorithm::HydraCountSketch, - HydraKind::HydraKll => SketchAlgorithm::HydraKll, - }) - } - FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => { - return Self::of_per_instance_sketch(kind.algorithm()) - } - FieldDataType::Plain(_) - | FieldDataType::SummarySample(..) - | FieldDataType::Wavelet(..) - | FieldDataType::StatModel(..) => (None, &[]), - }; + let (merge_inputs, readouts): (_, &'static [(StatisticKind, MergedError)]) = + match summary_type { + FieldDataType::ExactAggregate(kind, _) => match kind { + E::Sum | E::Count => (Some(Disjoint), &[]), + E::Min | E::Max => (Some(MayOverlap), &[]), + // The counter accumulator joins time-ordered windows with + // reset handling (`IncreaseAccumulator::merge_pair`). + E::Rate | E::Increase => (Some(DisjointInTime), &[]), + // IRate reads the last two samples of the window, which a + // tumbling window does not keep across its boundary. + E::IRate => (None, &[]), + }, + // A sketch algorithm's row is sketchlib's. HydraKll merges as + // that row says: `Hydra::merge` requires the same k and m and + // otherwise returns an error (asap_sketchlib #251). + FieldDataType::Sketch(algorithm) => { + let row = algorithm.capability(); + return Self { + merge_inputs: row.merge_inputs.map(MergeInputRule::from), + readouts: row.readouts, + }; + } + FieldDataType::Plain(_) + | FieldDataType::SummarySample(..) + | FieldDataType::Wavelet(..) + | FieldDataType::StatModel(..) => (None, &[]), + }; Self { merge_inputs, readouts, @@ -194,14 +186,7 @@ impl FamilyCapability { mod tests { use super::*; use crate::ir::scalar::expr_ir::ColumnRef; - use crate::ir::schema::{ExactParams, SketchKind, SketchParams}; - - fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::default(), - ) - } + use crate::summary_type::ExactParams; fn total() -> SketchStatistic { SketchStatistic::PointCount { @@ -221,16 +206,7 @@ mod tests { /// so it can serve all four; it answers no quantile or top-k. #[test] fn univmon_answers_its_total_and_every_frequency_moment() { - let univmon = FamilyCapability::of(&sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 256, - sketch_rows: 5, - sketch_cols: 1024, - layers: 16, - distinct_failure_probability: None, - }, - )); + let univmon = SummaryTypeCapability::of(&FieldDataType::Sketch(SketchAlgorithm::UnivMon)); for statistic in [ total(), SketchStatistic::Cardinality, @@ -252,40 +228,12 @@ mod tests { /// HLL answers the distinct count, not its total. #[test] fn summary_capability_facts() { - let kll = sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ); - let ddsketch = sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ); - let count_sketch = sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: 64, - depth: 5, - }, - ); - let cms_heap = sketch( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 5, - heap_size: 10, - }, - ); - let hll = sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, - ); - for (family, yes, no) in [ + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let ddsketch = FieldDataType::Sketch(SketchAlgorithm::DDSketch); + let count_sketch = FieldDataType::Sketch(SketchAlgorithm::CountSketch); + let cms_heap = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); + let hll = FieldDataType::Sketch(SketchAlgorithm::Hll); + for (summary_type, yes, no) in [ ( &kll, vec![ @@ -320,12 +268,18 @@ mod tests { vec![SketchStatistic::FrequencyL2, item(), total()], ), ] { - let capability = FamilyCapability::of(family); + let capability = SummaryTypeCapability::of(summary_type); for statistic in yes { - assert!(capability.answers(&statistic), "{family:?} {statistic:?}"); + assert!( + capability.answers(&statistic), + "{summary_type:?} {statistic:?}" + ); } for statistic in no { - assert!(!capability.answers(&statistic), "{family:?} {statistic:?}"); + assert!( + !capability.answers(&statistic), + "{summary_type:?} {statistic:?}" + ); } } } @@ -335,16 +289,9 @@ mod tests { /// per group: not counters, IRate or a Hydra. #[test] fn only_per_group_mergeable_families_merge_groups() { - use crate::ir::schema::{HydraKind, HydraParams}; - let kll = sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ); - assert!(FamilyCapability::merges_groups(&kll)); - assert!(FamilyCapability::merges_groups( + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + assert!(SummaryTypeCapability::merges_groups(&kll)); + assert!(SummaryTypeCapability::merges_groups( &FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) )); for (kind, params) in [ @@ -352,43 +299,23 @@ mod tests { (ExactKind::Increase, ExactParams::Increase), (ExactKind::IRate, ExactParams::IRate), ] { - assert!(!FamilyCapability::merges_groups( + assert!(!SummaryTypeCapability::merges_groups( &FieldDataType::ExactAggregate(kind, params) )); } - let FieldDataType::Sketch(kind, _) = sketch( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 64, - depth: 5, - }, - ) else { - unreachable!() - }; - let hydra = FieldDataType::Sketch( - kind, - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: 64, - depth: 5, - shared_rows: 3, - shared_columns: 256, - }, - }, - ); + let hydra = FieldDataType::Sketch(SketchAlgorithm::HydraCms); assert_eq!( - FamilyCapability::of(&hydra).merge_inputs, + SummaryTypeCapability::of(&hydra).merge_inputs, Some(MergeInputRule::Disjoint) ); - assert!(!FamilyCapability::merges_groups(&hydra)); + assert!(!SummaryTypeCapability::merges_groups(&hydra)); } /// An exact accumulator is read as its finalized value, not a readout. #[test] fn exact_accumulators_answer_no_readout() { let sum = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - assert!(FamilyCapability::of(&sum).readouts.is_empty()); - assert!(!FamilyCapability::of(&sum).answers(&total())); + assert!(SummaryTypeCapability::of(&sum).readouts.is_empty()); + assert!(!SummaryTypeCapability::of(&sum).answers(&total())); } } diff --git a/crates/types/src/summary_type/mod.rs b/crates/types/src/summary_type/mod.rs new file mode 100644 index 000000000..37c693d90 --- /dev/null +++ b/crates/types/src/summary_type/mod.rs @@ -0,0 +1,27 @@ +//! Summary types: each summary type's parameters $\boldsymbol\theta$ and +//! predefined parameter lists, what a summary is updated with and the +//! statistics read from it, and each summary type's capabilities. + +mod capability; +mod parameter_lists; +mod parameters; +mod statistic; + +/// A specific sketch algorithm — bounded error, sized by its +/// [`SketchParams`]. It is `asap_sketchlib`'s family enum; the statistics +/// each answers are its row of the family-capability table +/// ([`SummaryTypeCapability`]). The planner +/// sizes and offers only the families [`SketchParams`] has a variant for. +pub use asap_sketchlib::SketchAlgorithm; +pub use capability::{ + MergeInputRule, MergedError, StatisticKind, SummaryTypeCapability, SupportedSummary, +}; +pub use parameter_lists::parameter_list; +pub use parameters::{ + ExactKind, ExactParams, HydraParams, SamplingKind, SamplingParams, SketchParams, StatModelKind, + StatModelParams, WaveletKind, WaveletParams, EXACT_ACCUMULATOR_STATE_BYTES, KLL_MAX_K, +}; +pub use statistic::{ + EntityIdentity, NonNegativeWeightProof, SketchStatistic, SummaryInputExpr, SummaryUpdate, + WeightDomain, +}; diff --git a/crates/types/src/summary_type/parameter_lists.rs b/crates/types/src/summary_type/parameter_lists.rs new file mode 100644 index 000000000..c3ec86d4c --- /dev/null +++ b/crates/types/src/summary_type/parameter_lists.rs @@ -0,0 +1,12 @@ +//! Each summary algorithm's predefined parameter list $\boldsymbol\theta$, +//! beside [`SketchParams`]. + +use super::{SketchAlgorithm, SketchParams}; + +/// The algorithm's predefined $\boldsymbol\theta$ list: structures only; the +/// error of each, for a statistic at a failure probability, is its guarantee +/// (`asap_accuracy::theoretical_guarantee`). +pub fn parameter_list(algorithm: SketchAlgorithm) -> &'static [SketchParams] { + let _ = algorithm; + todo!("#1207 PR 1: the predefined θ list per algorithm") +} diff --git a/crates/types/src/summary_type/parameters.rs b/crates/types/src/summary_type/parameters.rs new file mode 100644 index 000000000..004855cc3 --- /dev/null +++ b/crates/types/src/summary_type/parameters.rs @@ -0,0 +1,235 @@ +//! Each summary type's parameters $\boldsymbol\theta$: structures only, +//! for exact accumulators, sketches, samples, wavelets, statistical models +//! and Hydra. + +use serde::{Deserialize, Serialize}; + +/// Bytes the executor holds per group for an exact accumulator of any +/// [`ExactKind`]: one element of its column of fixed-size states (the +/// executor's `ScalarState`, whose largest variant, a counter's, is 48 +/// bytes). Plan selection prices exact state with it; an executor test +/// asserts it equals that state's size. +pub const EXACT_ACCUMULATOR_STATE_BYTES: u64 = 56; + +/// An exact, mergeable accumulator family — zero approximation error. The +/// partial state built for one of these *is* the answer; no +/// `SummaryEstimate` evaluation is needed to get a value out of it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub enum ExactKind { + /// Exact sum accumulator (mergeable by addition). + Sum, + /// Exact count accumulator (mergeable by addition). + Count, + /// Exact minimum accumulator (mergeable by comparison). + Min, + /// Exact maximum accumulator (mergeable by comparison). + Max, + /// Exact increase accumulator (counter-reset-aware delta). + Increase, + /// Rate accumulator (increase / time window duration). + Rate, + /// Instant-rate accumulator retaining the final two timestamped samples. + IRate, +} + +/// Parameters for an [`ExactKind`] accumulator. All exact accumulators have +/// fixed semantics — no tuning parameters — so each variant carries none; +/// kept as a per-kind enum (mirroring [`SketchParams`]) so a mismatched +/// `(kind, params)` pair is still a type error, not a runtime check. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum ExactParams { + Sum, + Count, + Min, + Max, + Increase, + Rate, + IRate, +} + +/// The largest KLL `k` `asap_sketchlib` builds: it silently caps a larger +/// `k` at this (`MAX_CACHEABLE_K` in its `sketches::kll`). +pub const KLL_MAX_K: u32 = 26_602; + +/// The structure of a summary of one [`SketchAlgorithm`](crate::summary_type::SketchAlgorithm) (θ): its sizes, +/// from the algorithm's predefined list (`parameter_list`). No field records +/// the error or failure probability a configuration was sized for, so +/// equality compares every field. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum SketchParams { + UnivMon { + heap_size: u32, + sketch_rows: u32, + sketch_cols: u32, + layers: u8, + }, + Kll { + k: u32, + }, + Cms { + width: u32, + depth: u32, + }, + Hll { + precision: u8, + }, + DDSketch { + alpha: f64, + }, + CmsWithHeap { + width: u32, + depth: u32, + heap_size: u32, + }, + Kmv { + k: u32, + }, + CountSketch { + width: u32, + depth: u32, + }, + CountSketchWithHeap { + width: u32, + depth: u32, + heap_size: u32, + }, + /// The sizes of a Hydra structure shared across subpopulations, for the + /// `Hydra*` algorithms. + Hydra(HydraParams), +} + +/// A sampling summary family — retains an actual (weighted) subset of input +/// rows rather than a compressed sketch. The retained rows are its summary +/// sample ([`FieldDataType::SummarySample`](crate::ir::schema::FieldDataType::SummarySample)), +/// not the raw data samples a metric arrives as. Minimal +/// starting vocabulary: one family, extend as a deployment needs another +/// (stratified, weighted, …). +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub enum SamplingKind { + /// Reservoir sampling (mergeable via weighted reservoir merge; + /// uniform-random retained subset of a fixed size). + Reservoir, +} + +/// Parameters for a [`SamplingKind`] instance. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum SamplingParams { + Reservoir { + /// Reservoir capacity — the number of retained rows. + size: u32, + }, +} + +/// A wavelet-transform-based summary family — a compressed coefficient +/// vector supporting approximate range-sum / histogram queries. Minimal +/// starting vocabulary: one basis, extend as a deployment needs another. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub enum WaveletKind { + /// Haar wavelet synopsis — the simplest, most common streaming basis. + Haar, +} + +/// Parameters for a [`WaveletKind`] instance. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum WaveletParams { + Haar { + /// Number of retained coefficients — the compression / accuracy knob. + coefficients: u32, + }, +} + +/// A fitted statistical/parametric-model summary family — e.g. a distribution +/// fit used to answer approximate quantile/density queries without +/// retaining raw data samples. Deliberately open-ended: `family` names the model +/// family and is interpreted by whatever deployment builds/reads it, the +/// same "core doesn't enumerate every deployment shape" stance +/// `AggIntent::Extension` already takes for pre-ASAP intents. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub enum StatModelKind { + /// A parametric model fit to the data (e.g. a fitted distribution or + /// regression). `family` (in [`StatModelParams::Parametric`]) names + /// which one. + Parametric, +} + +/// Parameters for a [`StatModelKind`] instance. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum StatModelParams { + Parametric { + /// Deployment-interpreted model family name (e.g. + /// `"gaussian_mixture"`). Core has no fixed catalog of families — + /// see [`StatModelKind::Parametric`]. + family: String, + }, +} + +/// θ of the `Hydra*` algorithms ([`SketchAlgorithm::HydraKll`](crate::summary_type::SketchAlgorithm::HydraKll), +/// [`SketchAlgorithm::HydraCms`](crate::summary_type::SketchAlgorithm::HydraCms), [`SketchAlgorithm::HydraCountSketch`](crate::summary_type::SketchAlgorithm::HydraCountSketch)): one +/// structure shared by every subpopulation of a grouped aggregate (Manousis +/// et al., "Enabling Efficient and General Subpopulation Analytics in +/// Multidimensional Data Streams", VLDB 2022). It holds both the +/// per-subpopulation sketch's sizes and the shared grid's. HydraCms and +/// HydraCountSketch are the paper's proven construction (§4.5, Theorem 2); +/// HydraKll is not (the paper excludes quantiles, §4.3), so no error model +/// covers it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum HydraParams { + /// **No accuracy bound is modeled for this variant**. `k`/`shared_buckets` give a bound + /// sketch a concrete size to build against; unlike `HydraCms`/ + /// `HydraCountSketch`'s fields, they are not backed by the paper's + /// Theorem 2. + HydraKll { + /// Per-subpopulation accuracy knob — same meaning as + /// `SketchParams::Kll`'s own `k`, for the per-subpopulation logical + /// view this shared structure emulates. + k: u32, + /// Sizing knob for the one physical structure shared across every + /// subpopulation. Correctly sizing this against an estimated + /// subpopulation cardinality is a cost-model concern — out of scope + /// for the legality axis this type lives on — so this is + /// deliberately not derived from any cardinality estimate here. + shared_buckets: u32, + }, + /// Hydra over Count-Min Sketch — the paper's proven w×r + /// shared-grid construction (§4.2, Theorem 2). `width`/`depth` are the + /// per-subpopulation CMS knobs (mirroring `SketchParams::Cms`'s own + /// fields); `shared_rows`/`shared_columns` are the paper's `r`/`w` — the + /// redundant hash rows and shared-bucket width of the one physical grid + /// every subpopulation is hashed into. + HydraCms { + width: u32, + depth: u32, + /// The paper's `r`: redundant, pairwise-independent hash rows — + /// query time takes the median across rows to tighten the failure + /// probability (Theorem 2's `δ` term). + shared_rows: u32, + /// The paper's `w`: shared buckets per row that colliding + /// subpopulations share — the memory/accuracy knob Theorem 2's `ε` + /// term depends on. Sizing this against an estimated subpopulation + /// cardinality is a cost-model concern, deliberately out of scope + /// for the legality axis this type lives on. + shared_columns: u32, + }, + /// Hydra over Count-Sketch — same shape, and the same Theorem 2, as + /// `HydraCms`, over `SketchParams::CountSketch`'s knobs instead. + HydraCountSketch { + width: u32, + depth: u32, + shared_rows: u32, + shared_columns: u32, + }, +} + +impl HydraParams { + /// The parameters of the per-subpopulation sketch this shared structure + /// emulates, which its error bound and per-group state follow. + pub fn per_subpopulation(&self) -> SketchParams { + match *self { + HydraParams::HydraKll { k, .. } => SketchParams::Kll { k }, + HydraParams::HydraCms { width, depth, .. } => SketchParams::Cms { width, depth }, + HydraParams::HydraCountSketch { width, depth, .. } => { + SketchParams::CountSketch { width, depth } + } + } + } +} diff --git a/crates/types/src/summary_type/statistic.rs b/crates/types/src/summary_type/statistic.rs new file mode 100644 index 000000000..90e4d3aaf --- /dev/null +++ b/crates/types/src/summary_type/statistic.rs @@ -0,0 +1,213 @@ +//! What a summary is updated with and the statistics read from it. + +use serde::{Deserialize, Serialize}; + +use crate::ir::scalar::ColumnRef; + +/// One summary-state update. `item` identifies a keyed item when the family +/// is keyed; `weight` is the observation/update expression applied to state. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SummaryUpdate { + pub item: Option, + pub weight: SummaryInputExpr, + /// Planner proof for algorithms whose error/update semantics require a + /// non-negative stream. Absence of proof is never interpreted as zero or + /// non-negative. + #[serde(default)] + pub weight_domain: WeightDomain, +} + +impl SummaryUpdate { + pub fn column(column: ColumnRef) -> Self { + Self { + item: None, + weight: SummaryInputExpr::Column(column), + weight_domain: WeightDomain::UnknownOrSigned, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum WeightDomain { + #[default] + UnknownOrSigned, + NonNegative { + proof: NonNegativeWeightProof, + }, + /// An open requirement, not a proof: the weights are raw data samples of + /// `metric`, or plain sums of them (no rate, no other arithmetic), so they + /// are non-negative when every raw data sample of `metric` is: `metric` is + /// declared a counter, or declared non-negative. Logical optimization + /// records it without reading the data workload; plan selection confirms + /// it from the declared source facts + /// ([`SourceFacts::metric_type`](crate::workload::SourceFacts::metric_type), + /// [`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). + /// Unconfirmed, it counts as not proven. Serialized as + /// `samples_of_metric`. + #[serde(rename = "samples_of_metric")] + RawDataSamplesOfMetric { + metric: String, + }, + /// An open requirement, not a proof: the weights are reset-corrected + /// `rate`, `increase` or `irate` values of `metric`'s raw data samples, or plain + /// sums of them. Reset correction keeps them non-negative exactly when + /// the samples are, that is when `metric` is a counter; over a gauge a + /// negative sample can give a negative value. Plan selection confirms + /// it from the declared metric types, as + /// [`RawDataSamplesOfMetric`](Self::RawDataSamplesOfMetric). Unconfirmed, it + /// counts as not proven. + CounterDerivativeOfMetric { + metric: String, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NonNegativeWeightProof { + UnitCount, + /// `rate`, `increase` or `irate` of a counter's samples, or plain sums + /// of them: reset correction over non-negative samples gives a + /// non-negative increase, and a rate divides it by a positive duration. + /// Plan selection gives it on confirming a + /// [`WeightDomain::CounterDerivativeOfMetric`] requirement. + ResetAwareCounterDerivative, + /// Raw data samples of a metric declared a counter + /// ([`PrometheusMetricType::Counter`](crate::workload::PrometheusMetricType::Counter)), or + /// plain sums of them: a counter sample is never negative. Plan selection + /// gives it on confirming a + /// [`WeightDomain::RawDataSamplesOfMetric`] requirement. Serialized as + /// `counter_samples`. + #[serde(rename = "counter_samples")] + CounterRawDataSamples, + /// Raw data samples of a metric declared non-negative + /// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)), + /// such as a gauge, or plain sums of them. Plan selection gives it on + /// confirming a [`WeightDomain::RawDataSamplesOfMetric`] requirement for a + /// metric not declared a counter. Serialized as + /// `declared_non_negative_samples`. + #[serde(rename = "declared_non_negative_samples")] + DeclaredNonNegativeRawDataSamples, + /// A constant weight greater than 0, such as `Constant(2.0)`: every row + /// adds the same positive amount, so no weight is negative. + PositiveConstant, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum EntityIdentity { + PromqlLabelSet { excluding: Vec }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum SummaryInputExpr { + Constant(f64), + Column(ColumnRef), + Tuple(Vec), + EntityIdentity(EntityIdentity), +} + +/// The statistic computed from summary state by `SummaryEstimate`, for example +/// `Quantile { q: 0.99 }`. This is a result operation, not a workload query. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum SketchStatistic { + /// sqrt(sum_v frequency(v)^2), not the norm of numeric input values. + FrequencyL2, + /// Shannon entropy of the value-frequency distribution, in bits. + FrequencyEntropy, + /// Extract the value at quantile rank `q` ∈ (0, 1]. + Quantile { q: f64 }, + /// Estimated count / frequency. `key` names which column is being + /// queried (`ColumnRef::SampleValue` for the bare bucket total, with + /// `value: None`); a `Named`/`Qualified` `key` paired with + /// `value: Some(v)` is a per-item point lookup (e.g. + /// `count(cms_metric{item="checkout"})` — `key` is `item`, `value` is + /// `"checkout"`). `value` is carried here rather than resolved by the + /// `SummaryExecutor` from a `Filter` predicate because a summary + /// evaluation is read without access to the rest of the DAG. + PointCount { + key: ColumnRef, + value: Option, + }, + /// Estimated number of distinct elements. + Cardinality, + /// Read the top-k entries from a heap-backed summary. + TopK { k: usize }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keyed_summary_input_and_topk_evaluation_round_trip() { + for input in [ + SummaryUpdate { + item: Some(SummaryInputExpr::EntityIdentity( + EntityIdentity::PromqlLabelSet { excluding: vec![] }, + )), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::UnitCount, + }, + }, + SummaryUpdate { + item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( + ColumnRef::Named("service".into()), + )])), + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + weight_domain: WeightDomain::UnknownOrSigned, + }, + ] { + let input_json = serde_json::to_string(&input).unwrap(); + assert_eq!( + serde_json::from_str::(&input_json).unwrap(), + input + ); + let query = SketchStatistic::TopK { k: 10 }; + let json = serde_json::to_string(&query).unwrap(); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + query + ); + } + } + + /// The raw-data-sample and summary-sample renames keep their JSON names, + /// so committed plans and workloads still read. + #[test] + fn renamed_sample_terms_keep_their_wire_names() { + use crate::ir::schema::FieldDataType; + use crate::summary_type::{SamplingKind, SamplingParams}; + let json = |value: serde_json::Value| value.to_string(); + let raw = WeightDomain::RawDataSamplesOfMetric { + metric: "m".to_string(), + }; + assert_eq!( + serde_json::to_string(&raw).unwrap(), + json(serde_json::json!({"kind": "samples_of_metric", "metric": "m"})) + ); + for (proof, name) in [ + ( + NonNegativeWeightProof::CounterRawDataSamples, + "counter_samples", + ), + ( + NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples, + "declared_non_negative_samples", + ), + ] { + assert_eq!( + serde_json::to_string(&proof).unwrap(), + format!("\"{name}\"") + ); + } + let summary_sample = FieldDataType::SummarySample( + SamplingKind::Reservoir, + SamplingParams::Reservoir { size: 64 }, + ); + assert_eq!( + serde_json::to_string(&summary_sample).unwrap(), + json(serde_json::json!({"Sample": ["Reservoir", {"Reservoir": {"size": 64}}]})) + ); + } +} diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index d267ee3eb..202323407 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -2,45 +2,22 @@ //! crates' tests), exposed under the `test-support` feature so the normal //! public API stays as narrow as its production users need. Each forwards //! to the crate-private function of the same name. -use std::rc::Rc; -use crate::ir::materialized_view::RawDataWindow; use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; -use crate::ir::physical_export::PhysicalASAPDAGCompilation; use crate::ir::properties::summary_coverage::SummaryCoverage; -use crate::ir::properties::timing::NodeToTimedNodeCache; -use crate::ir::properties::ExecutionDataStateError; +use crate::plan::materialized_view::RawDataWindow; + use crate::ir::schema::Schema; use crate::ir::{ASAPOp, NonASAPOp}; use crate::ir::{OperatorNode, SchemaDerivationError}; -use crate::physical::Materialization; +use std::rc::Rc; /// `SummaryCoverage::definition_id`. pub fn definition_id(coverage: &SummaryCoverage) -> String { coverage.definition_id() } -/// `tumbling_window_chain::whole_window_build`. -pub fn whole_window_build(chain: &OperatorNode) -> Result { - crate::ir::operator::tumbling_window_chain::whole_window_build(chain) -} - -/// `tumbling_window_chain::tumbling_window`. -pub fn tumbling_window( - chain: &OperatorNode, - index: u32, -) -> Result { - crate::ir::operator::tumbling_window_chain::tumbling_window(chain, index) -} - -/// `physical_export::compile_physical_asap_dag_with_node_ids`. -pub fn compile_physical_asap_dag_with_node_ids( - root: &Rc, -) -> Result { - crate::ir::physical_export::compile_physical_asap_dag_with_node_ids(root) -} - /// `Operator::output_schema`. pub fn output_schema(operator: &Operator) -> Result { operator.output_schema() @@ -57,24 +34,42 @@ pub fn validate_asap_inputs(op: &ASAPOp) -> Result<(), SchemaDerivationError> { } /// `CurrentSeriesInput::matches_node`. -pub fn matches_node(input: &CurrentSeriesInput, node: &OperatorNode) -> bool { +pub fn matches_node(input: &CurrentSeriesInput, node: &Rc) -> bool { input.matches_node(node) } -/// `NodeToTimedNodeCache::timed`. -pub fn timed<'a>( - cache: &'a NodeToTimedNodeCache, - input: &Rc, -) -> Option<&'a Rc> { - cache.timed(input) -} - /// `RawDataWindow::lookback_ms`. pub fn lookback_ms(window: &RawDataWindow) -> u64 { window.lookback_ms() } -/// `Materialization::is_materialized_view`. -pub fn is_materialized_view(materialization: Materialization) -> bool { - materialization.is_materialized_view() +/// `physical_asap_dag::physical_asap_dag_at_query_time_with_node_ids`: `roots` +/// exported with every node at query time, not materialized and without θ, +/// with each exported node's operator node by node id. +pub fn physical_asap_dag_at_query_time_with_node_ids( + roots: &[crate::ir::QueryRoot], +) -> Result< + ( + crate::plan::physical_asap_dag::PhysicalASAPDAG, + Vec>, + ), + crate::plan::physical_asap_dag::PhysicalASAPDAGValidationError, +> { + crate::plan::physical_asap_dag::physical_asap_dag_at_query_time_with_node_ids(roots) +} + +/// [`physical_asap_dag_at_query_time_with_node_ids`] of the single root `root`, without +/// the node ids. +pub fn physical_asap_dag_at_query_time( + root: &Rc, +) -> Result< + crate::plan::physical_asap_dag::PhysicalASAPDAG, + crate::plan::physical_asap_dag::PhysicalASAPDAGValidationError, +> { + Ok( + physical_asap_dag_at_query_time_with_node_ids(&[crate::ir::QueryRoot::Operator( + Rc::clone(root), + )])? + .0, + ) } diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index a50afb0ed..4b17f2150 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -4,15 +4,14 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::rc::Rc; -use crate::ir::operator::operator_properties::{ - BinaryOpKind, PromQLVectorSetOpKind, RelationalSetOpKind, -}; +use crate::ir::operator::arguments::{BinaryOpKind, PromQLVectorSetOpKind, RelationalSetOpKind}; use crate::ir::operator::{AggIntent, GroupKeys, Reduction, Source}; -use crate::ir::physical_export::PhysicalASAPOperatorPayload as Payload; -use crate::ir::schema::{Schema, SketchStatistic, WeightDomain}; +use crate::ir::schema::Schema; use crate::ir::{ ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, QueryRoot, ScalarExpr, }; +use crate::plan::physical_asap_dag::PhysicalASAPOperatorPayload as Payload; +use crate::summary_type::SketchStatistic; use crate::workload::{DataWorkload, PrometheusMetricType}; /// A scan source's name: its PromQL metric or SQL table. @@ -208,7 +207,11 @@ fn declared_group_bound(node: &OperatorNode, data: &DataWorkload) -> Option ) => grouped(reduction, child), // A merge's children are states of one definition: its first // build's grouping stands for all of them. - Operator::ASAP(ASAPOp::SummaryMerge { .. }) => match node.summary_build() { + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + .. + }) => match children.first().and_then(|first| first.summary_build()) { Some( ASAPOp::SummaryAgg { reduction, child, .. @@ -216,7 +219,7 @@ fn declared_group_bound(node: &OperatorNode, data: &DataWorkload) -> Option | ASAPOp::SummaryTumblingWindowChain { reduction, child, .. }, - ) => grouped(reduction, child), + ) => grouped(&reduction, &child), _ => None, }, Operator::ASAP(ASAPOp::SummaryEstimate { @@ -365,11 +368,11 @@ fn at_most_one_row(node: &OperatorNode) -> bool { .. }) => keys.is_empty(), // A merge gives one state per group, whatever the tumbling windows it merges. - Operator::ASAP(ASAPOp::SummaryMerge { .. }) => matches!( - node.summary_build(), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => matches!( + children.first().and_then(|first| first.summary_build()), Some( ASAPOp::SummaryAgg { reduction, .. } | ASAPOp::SummaryTumblingWindowChain { reduction, .. } - ) if global(reduction) + ) if global(&reduction) ), Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, @@ -388,7 +391,7 @@ fn at_most_one_row(node: &OperatorNode) -> bool { Operator::NonASAP(NonASAPOp::Join { kind, left, right, .. }) => { - use crate::ir::operator::operator_properties::JoinKind; + use crate::ir::operator::arguments::JoinKind; match kind { JoinKind::Semi | JoinKind::Anti => at_most_one_row(left), JoinKind::Full => false, @@ -402,25 +405,14 @@ fn at_most_one_row(node: &OperatorNode) -> bool { } } -/// `domain` with an open requirement answered from `data`'s declared source -/// facts (`WeightDomain::confirmed`): samples are proven when the metric is -/// declared a counter or non-negative, a counter derivative only when it is -/// declared a counter; unknown otherwise. No type is inferred from a name. -pub fn confirmed_weights(domain: &WeightDomain, data: &DataWorkload) -> WeightDomain { - domain.confirmed( - |metric| declared_counter(data, metric), - |metric| declared_non_negative(data, metric), - ) -} - /// Whether `data` declares every sample of `metric` non-negative: the -/// metric is declared a counter, or a metric (a gauge, say) with -/// [`SourceFacts::values_non_negative`](crate::workload::SourceFacts::values_non_negative). +/// metric is declared a counter, or its `value` column is declared +/// non-negative ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). pub fn declared_non_negative(data: &DataWorkload, metric: &str) -> bool { declared_counter(data, metric) - || data - .series_facts(metric) - .is_some_and(|(facts, _)| facts.values_non_negative) + || data.series_facts(metric).is_some_and(|(facts, _)| { + facts.column_signs.get("value") == Some(&crate::stages::ValueSign::NonNegative) + }) } /// Whether `data` declares `metric` a counter, itself or as the `_sum` or @@ -432,72 +424,6 @@ fn declared_counter(data: &DataWorkload, metric: &str) -> bool { .is_some_and(|(_, series_type)| series_type == Some(PrometheusMetricType::Counter)) } -/// `roots` with every open [`WeightDomain::RawDataSamplesOfMetric`] requirement -/// replaced by its answer from `data` ([`confirmed_weights`]): the form of a -/// selected plan the executor runs, which reads no data workload. The same -/// `Rc` where nothing changes. -pub fn with_confirmed_weights(roots: &[QueryRoot], data: &DataWorkload) -> Vec { - with_confirmed_weights_at(roots, data, &|_| true) -} - -/// [`with_confirmed_weights`] at the summaries `confirm_at` names only: the -/// selected plan with the weight proofs Stage 3 made for its candidates -/// applied (`Selection::nodes_with_proven_non_negative_weights`). -pub fn with_confirmed_weights_at( - roots: &[QueryRoot], - data: &DataWorkload, - confirm_at: &dyn Fn(&Rc) -> bool, -) -> Vec { - fn confirm( - node: &Rc, - data: &DataWorkload, - confirm_at: &dyn Fn(&Rc) -> bool, - cache: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Rc::clone(done); - } - let mut changed = false; - let mut operator = node.operator.map_children(|child| { - let rebuilt = confirm(child, data, confirm_at, cache); - changed |= !Rc::ptr_eq(&rebuilt, child); - rebuilt - }); - if let Operator::ASAP( - ASAPOp::SummaryAgg { input, .. } | ASAPOp::SummaryTumblingWindowChain { input, .. }, - ) = &mut operator - { - let confirmed = confirmed_weights(&input.weight_domain, data); - if confirmed != input.weight_domain && confirm_at(node) { - input.weight_domain = confirmed; - changed = true; - } - } - let rebuilt = if changed { - Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - Rc::clone(node) - }; - cache.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); - rebuilt - } - let mut cache = HashMap::new(); - roots - .iter() - .map(|root| match root { - QueryRoot::Operator(n) => QueryRoot::Operator(confirm(n, data, confirm_at, &mut cache)), - QueryRoot::Scalar(expr) => QueryRoot::Scalar( - expr.map_operator_refs(&mut |n| confirm(n, data, confirm_at, &mut cache)), - ), - }) - .collect() -} - /// Rows per millisecond the sources in `sources` receive together, each /// at its own rate: `series_count` series each sampled every /// `sample_interval`, or, for a source that does not declare both, its @@ -548,7 +474,7 @@ pub fn source_rows_per_ms(sources: &BTreeSet<&str>, data: &DataWorkload) -> Opti /// declared label set never bounds values it does not describe. /// `producers` are each producer's output schema and unchanged columns. pub fn unchanged_source_columns<'a>( - node: &'a crate::ir::physical_export::PhysicalASAPDAGNode, + node: &'a crate::plan::physical_asap_dag::PhysicalASAPDAGNode, producers: &[(&'a Schema, &BTreeSet<&'a str>)], ) -> BTreeSet<&'a str> { let output_names = || node.output_schema.fields.iter().map(|f| f.name.as_str()); @@ -760,6 +686,7 @@ mod tests { let data = DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 4_000.0, + max_out_of_order_ms: None, }, sources: [("a", Some(100)), ("b", Some(300)), ("c", None)] .into_iter() diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs new file mode 100644 index 000000000..124580cd7 --- /dev/null +++ b/crates/types/src/workload/data.rs @@ -0,0 +1,796 @@ +//! The data workload: each source's arrival, size and declared facts. + +use std::collections::BTreeMap; + +use crate::stages::ValueSign; +use serde::{Deserialize, Serialize}; + +use super::{DurationMs, WorkloadError}; + +/// How the queried data arrives: already stored, still arriving, or both. +/// This is independent of whether queries repeat. +/// +/// JSON: `"at_rest"`, `{"ingesting": {"rows_per_second": 1000.0}}`, +/// `{"mixed": {"rows_per_second": 1000.0}}` or `"unknown"` (the default). +#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub enum DataArrival { + /// Every row is already stored; nothing arrives, so no summary can be + /// built at ingestion time. + AtRest, + /// Rows keep arriving while the queries run. + Ingesting { + /// Rows (PromQL samples) arriving per second over all sources. + /// Plan selection prices a source's scans with its share of it, by + /// series count when every source declares one, else with all of + /// it, when the source does not declare both its `series_count` and + /// its `sample_interval` ([`SourceFacts`]). + rows_per_second: f64, + /// How late a row may arrive relative to the newest row of its + /// series, in milliseconds. `None`: a PromQL source is taken to + /// arrive in order, and a SQL source gets no ingestion-time counter + /// build. + #[serde(default, skip_serializing_if = "Option::is_none")] + max_out_of_order_ms: Option, + }, + /// Older rows are stored and new rows keep arriving. + Mixed { + /// Rows (PromQL samples) arriving per second over all sources, as + /// for [`DataArrival::Ingesting`]. + rows_per_second: f64, + /// As for [`DataArrival::Ingesting`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + max_out_of_order_ms: Option, + }, + /// Not declared: no summary is built at ingestion time. + #[default] + Unknown, +} + +impl DataArrival { + /// Rows arriving per second, when the data is ingesting or mixed. + pub fn rows_per_second(&self) -> Option { + match self { + Self::Ingesting { + rows_per_second, .. + } + | Self::Mixed { + rows_per_second, .. + } => Some(*rows_per_second), + Self::AtRest | Self::Unknown => None, + } + } +} + +/// Queries per second, samples per second, or another rate whose unit is +/// established by the field that contains it. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Rate(pub f64); + +/// Facts about the data being queried: how it arrives, and what is known +/// about each source. Every fact is declared: the planner has no clock, so a +/// caller drops facts that no longer hold before planning. +/// +/// JSON: `{"arrival": {"ingesting": {"rows_per_second": 66666.7}}, +/// "sources": {"http_requests_total": {"series_count": 1000000, +/// "metric_type": "counter", "sample_interval": 15000}}}`. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DataWorkload { + /// How the data arrives (default: unknown). + pub arrival: DataArrival, + /// Facts per source, keyed by PromQL metric name or SQL table name + /// (default: none). A declared histogram or summary is keyed by its + /// family name; its `metric_type`, `column_signs` and + /// `sample_interval` apply to every series of the family + /// ([`Self::series_facts`]). Counts, norms and bounds (`series_count`, + /// `stream_norms`, …) differ per series name (a `foo_bucket` has one + /// series per `le`), so they are read from the series' own entry; + /// pricing a source's own arrival rate reads its `series_count` and + /// `sample_interval` there together. A source without an entry has no + /// known facts. + #[serde(default)] + pub sources: BTreeMap, +} + +/// What is known about one source (a PromQL metric or a SQL table). Every +/// fact is optional; an unknown fact is never read as zero. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceFacts { + /// How many series (PromQL) or groups of rows (SQL) the source holds + /// (default: unknown). Plan selection bounds the values a query returns + /// (`G`) by it, to split a whole-result failure probability, and prices + /// scans and selectors with it (with `sample_interval`, the source's + /// arrival rate). Unknown, pricing bounds a per-series count by the rows + /// read instead. Must be positive. + #[serde(default)] + pub series_count: Option, + /// Declared Prometheus type of the metric (`# TYPE` metadata). The + /// planner never infers a type from a name such as `_total` or + /// `_bucket`; an undeclared metric's samples may take any value. + #[serde(default)] + pub metric_type: Option, + /// The declared sign of each column's values, keyed by column name + /// (default: none declared), e.g. `{"value": "non_negative"}` for a + /// latency gauge. A `Scan`'s `column_sign` reads it; an undeclared + /// column is `Unknown`, except a PromQL counter's value, which is + /// non-negative without a declaration. The deployment must ensure it. + /// Omitted from JSON when empty. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub column_signs: BTreeMap, + /// How often each series of the metric (or group of rows of a SQL + /// table) receives a sample (default: unknown). A PromQL or MetricsQL + /// bare instant selector over the metric reads the latest sample within it; lowering such a + /// selector without it fails with + /// [`WorkloadError::MissingSampleInterval`]. With `series_count`, it + /// gives the source's own arrival rate, `series_count` samples per + /// `sample_interval`, which plan selection prices the source's scans + /// and time ranges at. + #[serde(default)] + pub sample_interval: Option, + /// Bounds on the source's stream norms: the inputs Count-Min and Count + /// Sketch additive error bounds are normalized by. Plan selection binds + /// them to turn those bounds into absolute errors; without them that + /// bound is unmet. + #[serde(default)] + pub stream_norms: Option, + /// Upper bound on the distinct items any one distinct-count summary over + /// the source's own rows counts in one complete evaluation, including + /// every merged tumbling window. One bound serves every query over the source, so + /// it must hold for the longest evaluation window any of them uses; a + /// filter or a shorter window only lowers it. Each group's own items + /// count separately: the bound is per output row. The deployment must + /// enforce it; an observed distinct count is not a bound. Plan + /// selection binds it to give classic HLL (and a distinct-count UnivMon) + /// its failure probability per output row; without it that probability + /// is unknown. KMV's guarantee needs no bound. + #[serde(default)] + pub max_distinct_per_evaluation: Option, + /// How many distinct values each label (PromQL) or column (SQL) of the + /// source takes, keyed by its name (default: none declared), e.g. + /// `{"host": 1000, "job": 10}`. Plan selection prices the groups of a + /// `by (k1, k2, ...)` at the product of its keys' counts, at most the + /// `series_count` (series or groups of rows) of the sources it reads + /// and the rows it reads per evaluation. No count is assumed for an + /// undeclared key: such a grouping is priced at the smaller of those + /// two. When every key of a + /// grouped result declares a count, their product (at most the series + /// read) also bounds the values one evaluation returns (`G`), which a + /// whole-result accuracy target splits its failure probability over: + /// each count must be an upper bound, counting a missing label (or a + /// NULL) as one more value, and positive. Omitted from JSON when empty. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub distinct_values_per_label: BTreeMap, + /// How many distinct combinations sets of labels (or columns) take + /// together, for labels that depend on each other (default: none + /// declared), e.g. `[{"labels": ["service", "endpoint"], + /// "distinct_values": 625}]` when each of 625 endpoints is in one of 25 + /// services. Plan selection prices a grouping at the smallest product + /// of declared counts whose labels cover its keys + /// (`SourceFacts::declared_distinct_combinations`), so + /// `by (service, endpoint)` is 625 groups rather than 25 x 625, and + /// `by (region, service, endpoint)` is at most 4 x 625. Nothing is + /// inferred from the per-label counts. Each set names two or more + /// distinct labels, and its count is a positive upper bound (a missing + /// label or NULL counts as one more value) no larger than the product + /// of its labels' `distinct_values_per_label` counts when all are + /// declared. Omitted from JSON when empty. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub distinct_values_per_label_set: Vec, + /// How many rows (PromQL samples) the source holds at rest, at most + /// (default: unknown). Plan selection bounds the rows a scan reads by + /// it: the whole source for a scan with no window above it, such as a + /// SQL query with no time predicate, and data at rest that declares no + /// arrival rate. Must be positive. Omitted from JSON when unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rows_held: Option, +} + +/// The distinct combinations of values a set of labels (or columns) of one +/// source takes together, an upper bound. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LabelSetDistinctValues { + /// The labels (PromQL) or columns (SQL), two or more, in any order. + pub labels: Vec, + /// How many distinct combinations of their values the source holds. + pub distinct_values: u64, +} + +/// The most keys declared label sets are searched over: covering them +/// searches every subset of the keys some declared set names. +const MAX_KEYS_COVERED_BY_LABEL_SETS: usize = 16; + +impl SourceFacts { + /// An upper bound on the distinct combinations of values `labels` take + /// together in this source, from declared counts only: the smallest + /// product of counts, of declared label sets + /// (`distinct_values_per_label_set`) and of single labels + /// (`distinct_values_per_label`), whose labels together cover + /// `labels`. A set covering labels beyond `labels` still bounds them. + /// `Some(1)` for no labels; `None` when some label is in no declared + /// count. Only labels some declared set names are searched over (every + /// other label contributes its own count); with more than 16 of them, + /// only the per-label product is used. + pub(crate) fn declared_distinct_combinations(&self, labels: &[&str]) -> Option { + let mut keys = labels.to_vec(); + keys.sort_unstable(); + keys.dedup(); + let label_product = |keys: &[&str]| { + keys.iter().try_fold(1u64, |product, key| { + Some(product.saturating_mul(*self.distinct_values_per_label.get(*key)?)) + }) + }; + let (in_sets, alone): (Vec<&str>, Vec<&str>) = keys.iter().partition(|key| { + self.distinct_values_per_label_set + .iter() + .any(|set| set.labels.iter().any(|label| label == *key)) + }); + if in_sets.is_empty() || in_sets.len() > MAX_KEYS_COVERED_BY_LABEL_SETS { + return label_product(&keys); + } + // Each declared count over `in_sets` as (the keys it covers as a bit + // mask, count). + let mut counts: Vec<(usize, u64)> = in_sets + .iter() + .enumerate() + .filter_map(|(bit, key)| Some((1 << bit, *self.distinct_values_per_label.get(*key)?))) + .collect(); + counts.extend(self.distinct_values_per_label_set.iter().filter_map(|set| { + let covered = in_sets + .iter() + .enumerate() + .filter(|(_, key)| set.labels.iter().any(|label| label == *key)) + .fold(0, |mask, (bit, _)| mask | 1 << bit); + (covered != 0).then_some((covered, set.distinct_values)) + })); + // smallest[m]: the smallest product of counts covering the keys in m. + let every_key = (1usize << in_sets.len()) - 1; + let mut smallest: Vec> = vec![None; every_key + 1]; + smallest[0] = Some(1); + for keys_left in 1..=every_key { + smallest[keys_left] = counts + .iter() + .filter(|(covered, _)| covered & keys_left != 0) + .filter_map(|(covered, count)| { + Some(smallest[keys_left & !covered]?.saturating_mul(*count)) + }) + .min(); + } + Some(smallest[every_key]?.saturating_mul(label_product(&alone)?)) + } +} + +/// Upper bounds on the norms of the frequency (weight) vector any one +/// summary evaluation over a source folds. A filter or a shorter window only +/// lowers them, so a bound for the whole source holds for every summary +/// over it. +#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StreamNorms { + /// `‖f‖₁`, the total weight: Count-Min's bound is `ε · ‖f‖₁`. + pub l1: Option, + /// `‖f‖₂`: Count Sketch's bound is `ε · ‖f‖₂`. + pub l2: Option, +} + +/// The Prometheus type of a metric family (`# TYPE` metadata). Not an ASAP +/// summary: [`Self::Summary`] is the Prometheus summary type. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrometheusMetricType { + /// Starts at zero and only increases, or resets to zero: every sample is + /// non-negative. + Counter, + /// May take any value. + Gauge, + /// A classic histogram family `foo`: cumulative bucket counters + /// `foo_bucket{le=…}`, and the counters `foo_sum` and `foo_count`. The + /// planner reads `foo_sum` as a counter, so it assumes the family's + /// observations are non-negative. + Histogram, + /// A native histogram: its samples are histograms, which the IR cannot + /// represent yet, so a query reading one is rejected. + NativeHistogram, + /// A Prometheus summary family `foo`: quantiles each client computed per + /// series, `foo{quantile="φ"}`, and the counters `foo_sum` and + /// `foo_count`. + Summary, +} + +impl DataWorkload { + /// The facts of the source a PromQL selector over `series` reads, and + /// that series' own type. A declared histogram family `foo` holds the + /// series `foo_bucket`, `foo_sum` and `foo_count`; a declared summary + /// family `foo` holds `foo`, `foo_sum` and `foo_count`; `_sum` and + /// `_count` series are counters. A declared family wins over a series' + /// own entry (`foo_sum`'s entry is ignored here when `foo` is a declared + /// histogram). Any other name is its own source, looked up as is: a + /// suffix never makes a type. + pub fn series_facts( + &self, + series: &str, + ) -> Option<(&SourceFacts, Option)> { + use PrometheusMetricType::{Counter, Histogram, Summary}; + for (suffix, series_type) in [ + ("_bucket", None), + ("_sum", Some(Counter)), + ("_count", Some(Counter)), + ] { + let Some(family) = series.strip_suffix(suffix) else { + continue; + }; + let Some(facts) = self.sources.get(family) else { + continue; + }; + match (facts.metric_type, series_type) { + (Some(Histogram), None) => return Some((facts, Some(Histogram))), + (Some(Histogram | Summary), Some(counter)) => return Some((facts, Some(counter))), + _ => {} + } + } + let facts = self.sources.get(series)?; + Some((facts, facts.metric_type)) + } + + /// Data of unknown arrival whose `metrics` are each sampled every + /// `sample_interval`, with no other facts: what PromQL or MetricsQL + /// queries over them need to lower their bare instant selectors. + pub fn sampled_every( + metrics: impl IntoIterator, + sample_interval: DurationMs, + ) -> Self { + let sampled = SourceFacts { + sample_interval: Some(sample_interval), + ..SourceFacts::default() + }; + Self { + arrival: DataArrival::Unknown, + sources: metrics + .into_iter() + .map(|metric| (metric, sampled.clone())) + .collect(), + } + } + + /// This data workload with every source it declares holding + /// `series_count` series (or groups of rows). + #[cfg(any(test, feature = "test-support"))] + pub fn with_series_count(mut self, series_count: u64) -> Self { + for facts in self.sources.values_mut() { + facts.series_count = Some(series_count); + } + self + } + + /// Validate the arrival and source facts independently of query demand + /// or language. + pub fn validate(&self) -> Result<(), WorkloadError> { + validate_optional_rate(self.arrival.rows_per_second().map(Rate))?; + for (source_name, facts) in &self.sources { + if facts.series_count == Some(0) { + return Err(WorkloadError::ZeroSeriesCount); + } + if facts.sample_interval == Some(DurationMs(0)) { + return Err(WorkloadError::ZeroSampleInterval); + } + if let Some(norms) = &facts.stream_norms { + for value in [norms.l1, norms.l2].into_iter().flatten() { + if !(value.is_finite() && value >= 0.0) { + return Err(WorkloadError::InvalidStreamNorm(value)); + } + } + } + if facts.max_distinct_per_evaluation == Some(0) { + return Err(WorkloadError::ZeroMaxDistinct); + } + if facts + .distinct_values_per_label + .values() + .any(|&count| count == 0) + { + return Err(WorkloadError::ZeroDistinctValues); + } + if facts.rows_held == Some(0) { + return Err(WorkloadError::ZeroRowsHeld); + } + for set in &facts.distinct_values_per_label_set { + let error_labels = || set.labels.clone(); + let mut distinct_labels: Vec<&String> = set.labels.iter().collect(); + distinct_labels.sort_unstable(); + distinct_labels.dedup(); + if distinct_labels.len() < 2 || distinct_labels.len() != set.labels.len() { + return Err(WorkloadError::LabelSetNeedsTwoLabels { + source_name: source_name.clone(), + labels: error_labels(), + }); + } + if set.distinct_values == 0 { + return Err(WorkloadError::ZeroDistinctValues); + } + let label_product = set.labels.iter().try_fold(1u64, |product, label| { + Some(product.saturating_mul(*facts.distinct_values_per_label.get(label)?)) + }); + if let Some(label_product) = label_product.filter(|&p| set.distinct_values > p) { + return Err(WorkloadError::LabelSetCountAboveLabelProduct { + source_name: source_name.clone(), + labels: error_labels(), + distinct_values: set.distinct_values, + label_product, + }); + } + } + } + Ok(()) + } +} + +fn validate_optional_rate(rate: Option) -> Result<(), WorkloadError> { + rate.map(validate_rate).transpose().map(|_| ()) +} + +pub(super) fn validate_rate(rate: Rate) -> Result { + if rate.0.is_finite() && rate.0 >= 0.0 { + Ok(rate) + } else { + Err(WorkloadError::InvalidRate(rate.0)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workload::*; + + /// A stream norm is a finite, non-negative magnitude. + #[test] + fn stream_norms_must_be_finite_and_non_negative() { + let with = |l1: f64| DataWorkload { + sources: BTreeMap::from([( + "m".to_string(), + SourceFacts { + stream_norms: Some(StreamNorms { + l1: Some(l1), + l2: None, + }), + ..SourceFacts::default() + }, + )]), + ..DataWorkload::default() + }; + assert!(with(10.0).validate().is_ok()); + for bad in [-1.0, f64::NAN, f64::INFINITY] { + assert!( + matches!( + with(bad).validate(), + Err(WorkloadError::InvalidStreamNorm(_)) + ), + "{bad}" + ); + } + } + + /// A distinct-count bound is positive: a summary counts at least one item. + #[test] + fn max_distinct_per_evaluation_must_be_positive() { + let with = |bound: u64| DataWorkload { + sources: BTreeMap::from([( + "m".to_string(), + SourceFacts { + max_distinct_per_evaluation: Some(bound), + ..SourceFacts::default() + }, + )]), + ..DataWorkload::default() + }; + assert!(with(1).validate().is_ok()); + assert_eq!(with(0).validate(), Err(WorkloadError::ZeroMaxDistinct)); + } + + /// A declared `rows_held` is positive. + #[test] + fn rows_held_must_be_positive() { + let with = |rows: u64| DataWorkload { + sources: BTreeMap::from([( + "t".to_string(), + SourceFacts { + rows_held: Some(rows), + ..SourceFacts::default() + }, + )]), + ..DataWorkload::default() + }; + assert!(with(1).validate().is_ok()); + assert_eq!(with(0).validate(), Err(WorkloadError::ZeroRowsHeld)); + } + + /// A declared distinct-value count is positive: a label with no value + /// would make every grouping over it empty. + #[test] + fn distinct_values_per_label_must_be_positive() { + let with = |count: u64| DataWorkload { + sources: BTreeMap::from([( + "m".to_string(), + SourceFacts { + distinct_values_per_label: BTreeMap::from([ + ("host".to_string(), 1_000), + ("job".to_string(), count), + ]), + ..SourceFacts::default() + }, + )]), + ..DataWorkload::default() + }; + assert!(with(1).validate().is_ok()); + assert_eq!(with(0).validate(), Err(WorkloadError::ZeroDistinctValues)); + } + + /// The `http` source of the eval's RollupHeavy workload: 4 regions, 25 + /// services and 625 endpoints, each endpoint in one service, so + /// (service, endpoint) takes 625 combinations, not 25 x 625. + fn hierarchical_facts(declared_sets: &[(&[&str], u64)]) -> SourceFacts { + SourceFacts { + distinct_values_per_label: BTreeMap::from([ + ("region".to_string(), 4), + ("service".to_string(), 25), + ("endpoint".to_string(), 625), + ]), + distinct_values_per_label_set: declared_sets + .iter() + .map(|(labels, distinct_values)| LabelSetDistinctValues { + labels: labels.iter().map(|label| label.to_string()).collect(), + distinct_values: *distinct_values, + }) + .collect(), + ..SourceFacts::default() + } + } + + /// A grouping's combinations are bounded by the smallest product over + /// declared sets and labels covering its keys: the declared set itself, + /// a superset of keys through the set times the other labels, and a + /// subset of a declared set through that set. Without a declared set, + /// the per-label product stands. + #[test] + fn declared_label_sets_bound_the_combinations_of_their_labels() { + let declared = hierarchical_facts(&[(&["service", "endpoint"], 625)]); + let undeclared = hierarchical_facts(&[]); + let combinations = + |facts: &SourceFacts, labels: &[&str]| facts.declared_distinct_combinations(labels); + assert_eq!(combinations(&declared, &["service", "endpoint"]), Some(625)); + assert_eq!(combinations(&declared, &["endpoint", "service"]), Some(625)); + assert_eq!( + combinations(&declared, &["region", "service", "endpoint"]), + Some(4 * 625) + ); + assert_eq!( + combinations(&undeclared, &["service", "endpoint"]), + Some(25 * 625) + ); + assert_eq!( + combinations(&undeclared, &["region", "service", "endpoint"]), + Some(4 * 25 * 625) + ); + assert_eq!(combinations(&declared, &["service"]), Some(25)); + assert_eq!(combinations(&declared, &[]), Some(1)); + // A key no fact covers leaves the combinations unbound. + assert_eq!(combinations(&declared, &["service", "status"]), None); + // A declared set covering more labels bounds its subsets: 3 users + // have made 7 (user, session) pairs, so at most 7 sessions, while + // `session` alone declares no count. + let sessions = SourceFacts { + distinct_values_per_label_set: vec![LabelSetDistinctValues { + labels: vec!["user".to_string(), "session".to_string()], + distinct_values: 7, + }], + ..SourceFacts::default() + }; + assert_eq!( + sessions.declared_distinct_combinations(&["session"]), + Some(7) + ); + // 70 keys, two in a declared set: only those two are searched over. + let mut wide = hierarchical_facts(&[(&["service", "endpoint"], 625)]); + let names: Vec = (0..68).map(|i| format!("l{i}")).collect(); + for name in &names { + wide.distinct_values_per_label.insert(name.clone(), 1); + } + let mut labels: Vec<&str> = names.iter().map(String::as_str).collect(); + labels.extend(["service", "endpoint"]); + assert_eq!(wide.declared_distinct_combinations(&labels), Some(625)); + } + + /// A declared set count is an input error, naming the source, when it + /// is above the product of its labels' declared counts, or when the set + /// has fewer than two distinct labels (one label's count belongs in + /// `distinct_values_per_label`) or zero values. + #[test] + fn a_label_set_count_is_at_most_its_labels_product() { + let with = |facts: SourceFacts| DataWorkload { + sources: BTreeMap::from([("http".to_string(), facts)]), + ..DataWorkload::default() + }; + assert!(with(hierarchical_facts(&[(&["service", "endpoint"], 625)])) + .validate() + .is_ok()); + // `status` declares no count, so the set's product is unknown. + assert!(with(hierarchical_facts(&[(&["service", "status"], 1_000)])) + .validate() + .is_ok()); + let error = with(hierarchical_facts(&[(&["region", "service"], 101)])) + .validate() + .unwrap_err(); + assert_eq!( + error, + WorkloadError::LabelSetCountAboveLabelProduct { + source_name: "http".to_string(), + labels: vec!["region".to_string(), "service".to_string()], + distinct_values: 101, + label_product: 100, + } + ); + assert!(error.to_string().contains("http"), "{error}"); + for labels in [&["service"][..], &["service", "service"][..]] { + let error = with(hierarchical_facts(&[(labels, 25)])).validate(); + assert!( + matches!( + &error, + Err(WorkloadError::LabelSetNeedsTwoLabels { source_name, .. }) + if source_name == "http" + ), + "{error:?}" + ); + } + assert_eq!( + with(hierarchical_facts(&[(&["service", "endpoint"], 0)])).validate(), + Err(WorkloadError::ZeroDistinctValues) + ); + } + + /// A declared series count is positive: with zero series, `absent(m)` + /// would still return one row the bound does not count. + #[test] + fn series_count_must_be_positive() { + let with = |series_count: u64| DataWorkload { + sources: BTreeMap::from([( + "m".to_string(), + SourceFacts { + series_count: Some(series_count), + ..SourceFacts::default() + }, + )]), + ..DataWorkload::default() + }; + assert!(with(1).validate().is_ok()); + assert_eq!(with(0).validate(), Err(WorkloadError::ZeroSeriesCount)); + } + + /// The arrival's rate is finite and non-negative, and each source's + /// declared sample interval is positive. + #[test] + fn arrival_rate_and_sample_interval_are_validated() { + let with = |rows_per_second: f64, sample_interval: Option| DataWorkload { + arrival: DataArrival::Mixed { + rows_per_second, + max_out_of_order_ms: None, + }, + sources: BTreeMap::from([( + "m".to_string(), + SourceFacts { + sample_interval: sample_interval.map(DurationMs), + ..SourceFacts::default() + }, + )]), + }; + assert!(with(10.0, Some(15_000)).validate().is_ok()); + assert!(with(0.0, None).validate().is_ok()); + assert_eq!( + with(-1.0, None).validate(), + Err(WorkloadError::InvalidRate(-1.0)) + ); + for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + assert!( + matches!( + with(bad, None).validate(), + Err(WorkloadError::InvalidRate(_)) + ), + "{bad}" + ); + } + assert_eq!( + with(1.0, Some(0)).validate(), + Err(WorkloadError::ZeroSampleInterval) + ); + } + + /// The former flat shape (per-field evidence, one global cardinality, + /// per-fact source maps, `continuously_ingesting`) is rejected, as are + /// freshness evidence on a source and a sample interval on the arrival. + #[test] + fn the_former_data_workload_shape_is_rejected() { + let declared = |value: serde_json::Value| { + serde_json::json!({"value": value, "source": "declared", + "observed_at_ms": null, "valid_for_ms": null}) + }; + for old in [ + serde_json::json!({ + "arrival": "continuously_ingesting", + "data_ingestion_interval": declared(15000.into()), + "ingestion_rate": declared(1000.0.into()), + "input_cardinality": declared(10000.into()), + }), + serde_json::json!({"arrival": "continuously_ingesting"}), + serde_json::json!({"arrival": "at_rest", "input_cardinality": declared(1.into())}), + serde_json::json!({"arrival": "at_rest", + "metric_types": {"m": "counter"}}), + serde_json::json!({"arrival": "at_rest", + "sources": {"m": {"series_count": 1, "evidence": {"observed_at_ms": 1}}}}), + serde_json::json!({"arrival": "at_rest", "sources": {"m": {"series": 1}}}), + serde_json::json!({"arrival": {"ingesting": + {"rows_per_second": 1.0, "sample_interval": 15000}}}), + serde_json::json!({"arrival": {"at_rest": {"rows_per_second": 1.0}}}), + ] { + assert!( + serde_json::from_value::(old.clone()).is_err(), + "{old}" + ); + } + } + + /// `metric_type` takes each Prometheus type by its snake_case name. + #[test] + fn prometheus_metric_types_have_snake_case_wire_names() { + for (metric_type, name) in [ + (PrometheusMetricType::Counter, "counter"), + (PrometheusMetricType::Gauge, "gauge"), + (PrometheusMetricType::Histogram, "histogram"), + (PrometheusMetricType::NativeHistogram, "native_histogram"), + (PrometheusMetricType::Summary, "summary"), + ] { + let facts: SourceFacts = + serde_json::from_value(serde_json::json!({ "metric_type": name })).unwrap(); + assert_eq!(facts.metric_type, Some(metric_type)); + } + } + + /// A declared histogram family supplies its `_bucket`, `_sum` and + /// `_count` series, a declared summary family its `_sum` and `_count` + /// series, with `_sum` and `_count` counters; any other name, including + /// a suffixed one whose family is undeclared or a counter, is looked up + /// as is. + #[test] + fn series_facts_resolve_declared_histogram_and_summary_families() { + use PrometheusMetricType::{Counter, Gauge, Histogram, Summary}; + let facts = |metric_type| SourceFacts { + metric_type: Some(metric_type), + sample_interval: Some(DurationMs(15_000)), + ..SourceFacts::default() + }; + let data = DataWorkload { + arrival: DataArrival::Unknown, + sources: BTreeMap::from([ + ("h".to_string(), facts(Histogram)), + ("s".to_string(), facts(Summary)), + ("c".to_string(), facts(Counter)), + ("g_sum".to_string(), facts(Gauge)), + ]), + }; + let series_type = |series: &str| data.series_facts(series).map(|(_, t)| t); + assert_eq!(series_type("h"), Some(Some(Histogram))); + assert_eq!(series_type("h_bucket"), Some(Some(Histogram))); + assert_eq!(series_type("h_sum"), Some(Some(Counter))); + assert_eq!(series_type("h_count"), Some(Some(Counter))); + assert_eq!(series_type("s"), Some(Some(Summary))); + assert_eq!(series_type("s_sum"), Some(Some(Counter))); + assert_eq!(series_type("s_count"), Some(Some(Counter))); + assert_eq!(series_type("s_bucket"), None); + assert_eq!(series_type("c_sum"), None); + assert_eq!(series_type("g_sum"), Some(Some(Gauge))); + assert_eq!( + data.series_facts("h_sum").unwrap().0.sample_interval, + Some(DurationMs(15_000)) + ); + } +} diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index fe7dec720..5b4f20270 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -1,1826 +1,20 @@ +//! The workload the planner plans for, by scope: the query workload +//! (`query`), the data workload (`data`), the two planned together +//! (`planning`), and the bounds read from them ([`bounds`]). + pub mod bounds; +mod data; pub mod parsed_workload; - -use std::collections::BTreeMap; - -use crate::ir::properties::AccuracyTarget; -use serde::{Deserialize, Serialize}; - -// ── Query surface ───────────────────────────────────────────────────────────── - -/// A raw query string in its source language, before any parsing. -#[derive(Debug, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct Query(pub String); - -/// How often a repeating query fires, in milliseconds. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct RepetitionInterval(pub u32); - -/// Milliseconds since the Unix epoch. Workload timestamps use one explicit -/// representation so schedules, observations, and time selections agree. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct TimestampMs(pub u64); - -/// A non-negative duration in milliseconds. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct DurationMs(pub u64); - -/// SQL dialect variant — different dialects have different syntax and -/// function sets that affect how the query string is parsed. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum SqlDialect { - #[serde(rename = "datafusion_sql")] - DataFusionSQL, - #[serde(rename = "clickhouse_sql")] - ClickhouseSQL, - #[serde(rename = "elastic_sql")] - ElasticSQL, -} - -/// Source language of every query in the workload. -/// All queries in a single `QueryWorkload` share the same language. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum QueryLanguage { - #[serde(rename = "promql")] - PromQL, - #[serde(rename = "metricsql")] - MetricsQL, - #[serde(rename = "sql")] - SQL(SqlDialect), - #[serde(rename = "datafusion")] - DataFusion, - #[serde(rename = "elastic_dsl")] - ElasticDSL, -} - -// ── Per-query requirements ──────────────────────────────────────────────────── - -/// Independent accuracy and response-latency constraints attached to one -/// query in the workload. -/// -/// JSON: `{"accuracy": {"max_error": 0.01, "accuracy_metric": "rank"}, "max_latency_ms": -/// 200.0}`. An omitted `accuracy` is [`AccuracyTarget::exact`] and an -/// omitted `max_latency_ms` imposes no bound. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct QueryRequirements { - /// The accuracy the query's answer must meet. - #[serde(default = "AccuracyTarget::exact")] - pub accuracy: AccuracyTarget, - /// The longest one evaluation may take, in milliseconds; `None` for no - /// bound. - #[serde(default)] - pub max_latency_ms: Option, -} - -impl Default for QueryRequirements { - fn default() -> Self { - Self { - accuracy: AccuracyTarget::exact(), - max_latency_ms: None, - } - } -} - -// ── Workload entries ────────────────────────────────────────────────────────── - -/// How far in advance the planner knows that a query will be executed. -/// This describes knowledge of the query, not how often it runs. -/// -/// # Example -/// -/// A report announced at `1_000` ms and executed later is predictable; -/// an interactive query typed by a user is ad hoc. -/// -/// ``` -/// use asap_types::workload::{Predictability, TimestampMs}; -/// -/// let report = Predictability::Predictable { -/// known_at: Some(TimestampMs(1_000)), -/// }; -/// let exploration = Predictability::AdHoc; -/// ``` -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case", deny_unknown_fields)] -pub enum Predictability { - AdHoc, - Predictable { - known_at: Option, - }, - #[default] - Unknown, -} - -/// Evidence-backed forecast used when recurrence is not a fixed interval or -/// an explicit schedule. Demand is normalized to invocations per second at -/// the input boundary, so this type never mixes counts and rates. -/// -/// # Example -/// -/// This estimate predicts ten executions per second. It is usable for five -/// minutes after `observed_at`. -/// -/// ``` -/// use asap_types::workload::{DemandEstimate, DurationMs, Rate, TimestampMs}; -/// -/// let estimate = DemandEstimate { -/// expected_rate: Rate(10.0), -/// observed_at: Some(TimestampMs(60_000)), -/// valid_for: Some(DurationMs(5 * 60 * 1_000)), -/// }; -/// ``` -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct DemandEstimate { - pub expected_rate: Rate, - pub observed_at: Option, - pub valid_for: Option, -} - -/// How a repeated query is expected to recur. -/// -/// # Example -/// -/// ``` -/// use asap_types::workload::{RepeatedDemand, RepetitionInterval, TimestampMs}; -/// -/// let dashboard = RepeatedDemand::FixedInterval(RepetitionInterval(10_000)); -/// let scheduled = RepeatedDemand::Scheduled(vec![ -/// TimestampMs(100_000), -/// TimestampMs(200_000), -/// ]); -/// ``` -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RepeatedDemand { - FixedInterval(RepetitionInterval), - /// Fixed cadence with a known evaluation phase. The timestamp is any - /// evaluation endpoint on that cadence, not necessarily its first run. - FixedIntervalAt { - interval: RepetitionInterval, - evaluation_phase: TimestampMs, - }, - Scheduled(Vec), - EstimatedRate(DemandEstimate), -} - -/// Normalized execution recurrence shared by batch and repeating workload -/// entries. This is independent of [`Predictability`]: a one-time query may -/// be known in advance or ad hoc, and a repeated query may still be uncertain. -/// -/// # Example -/// -/// ``` -/// use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval, TimestampMs}; -/// -/// let scheduled_once = QueryRecurrence::OneTime { -/// invocations: 1, -/// execute_at: Some(TimestampMs(100_000)), -/// }; -/// let every_minute = QueryRecurrence::Repeated( -/// RepeatedDemand::FixedInterval(RepetitionInterval(60_000)), -/// ); -/// ``` -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case", deny_unknown_fields)] -pub enum QueryRecurrence { - OneTime { - invocations: u64, - execute_at: Option, - }, - Repeated(RepeatedDemand), - #[default] - Unknown, -} - -impl QueryRecurrence { - /// The fixed interval at which the query repeats, in milliseconds: - /// `Some` only for a fixed-interval repetition with a positive interval. - pub fn fixed_interval_ms(&self) -> Option { - match self { - QueryRecurrence::Repeated( - RepeatedDemand::FixedInterval(interval) - | RepeatedDemand::FixedIntervalAt { interval, .. }, - ) => Some(u64::from(interval.0)).filter(|&ms| ms > 0), - _ => None, - } - } -} - -/// The most evaluation steps one range request may have, as Prometheus -/// allows for `/api/v1/query_range` (it rejects a request with more). -pub(crate) const MAX_RANGE_EVALUATION_STEPS: u64 = 11_000; - -/// How one request of a query is answered as a range request, as -/// Prometheus' `/api/v1/query_range` does: it evaluates the query at -/// `start = end − span_ms` and every `step_ms` after, up to and including -/// the request's `end`, e.g. a dashboard panel over the last hour at -/// 1-minute steps (`...&start=now-1h&end=now&step=60`). `span_ms` must be a -/// whole number of steps, so the last step is the request's end. -/// [`QueryRecurrence`] still says how often a request runs. -/// -/// JSON: `{"step_ms": 60000, "span_ms": 3600000}`. -/// -/// # Example -/// -/// ``` -/// use asap_types::workload::RangeEvaluation; -/// -/// let last_hour_by_minute = RangeEvaluation { step_ms: 60_000, span_ms: 3_600_000 }; -/// assert_eq!(last_hour_by_minute.evaluations_per_request(), 61); -/// ``` -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct RangeEvaluation { - /// Time between consecutive evaluation steps of one request, in - /// milliseconds; must be positive. - pub step_ms: u64, - /// How far before the request's end its first step is, in - /// milliseconds: a multiple of `step_ms`, 0 for one step per request. - pub span_ms: u64, -} - -impl RangeEvaluation { - /// Steps one request evaluates, `span_ms / step_ms + 1`: 61 for a 1 h - /// span at 1 min steps. Stage 3 prices each step as one evaluation. - pub fn evaluations_per_request(&self) -> u64 { - self.span_ms / self.step_ms.max(1) + 1 - } -} - -/// The greatest common divisor of `a` and `b` (`a` when `b` is 0): the -/// grid two cadences in milliseconds share. -pub fn greatest_common_divisor(a: u64, b: u64) -> u64 { - if b == 0 { - a - } else { - greatest_common_divisor(b, a % b) - } -} - -/// One query submitted as part of a finite batch, including how many times it -/// will run and, when known, its execution time. -/// -/// # Example -/// -/// ``` -/// use asap_types::workload::*; -/// -/// let report = BatchEntry { -/// query: Query("SELECT count(*) FROM events".into()), -/// requirements: QueryRequirements::default(), -/// predictability: Predictability::Predictable { -/// known_at: Some(TimestampMs(1_000)), -/// }, -/// invocations: 1, -/// execute_at: Some(TimestampMs(10_000)), -/// range_evaluation: None, -/// }; -/// ``` -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct BatchEntry { - pub query: Query, - pub requirements: QueryRequirements, - pub predictability: Predictability, - pub invocations: u64, - pub execute_at: Option, - /// How each invocation is answered as a range request: `None` (the - /// default) for one evaluation per invocation. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub range_evaluation: Option, -} - -/// One query with repeated demand. Its recurrence does not imply that the -/// queried data is continuously ingesting; that is described separately by -/// [`DataArrival`]. -/// -/// # Example -/// -/// ``` -/// use asap_types::workload::*; -/// -/// let dashboard = RepeatingEntry { -/// query: Query("rate(requests[5m])".into()), -/// demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), -/// requirements: QueryRequirements::default(), -/// predictability: Predictability::Predictable { known_at: None }, -/// // Each refresh evaluates the last hour at 1-minute steps. -/// range_evaluation: Some(RangeEvaluation { step_ms: 60_000, span_ms: 3_600_000 }), -/// }; -/// ``` -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct RepeatingEntry { - pub query: Query, - pub demand: RepeatedDemand, - pub requirements: QueryRequirements, - pub predictability: Predictability, - /// How each request is answered as a range request: `None` (the - /// default) for one evaluation per request. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub range_evaluation: Option, -} - -/// Planner-facing normalized form of either [`BatchEntry`] or -/// [`RepeatingEntry`]. It keeps requirements, predictability and recurrence -/// as separate axes; the event time a query reads is in its text. Planning takes one per query, in -/// root order: Stage 2 (physical ASAP-aware optimization) reads the -/// recurrence and predictability, Stage 3 (plan selection) the accuracy -/// target, the latency bound and the recurrence. -/// -/// # Example -/// -/// ``` -/// use asap_types::workload::*; -/// -/// let entry = QueryWorkloadEntry { -/// query: Query("rate(requests[5m])".into()), -/// requirements: QueryRequirements::default(), -/// predictability: Predictability::Predictable { known_at: None }, -/// recurrence: QueryRecurrence::Repeated( -/// RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), -/// ), -/// range_evaluation: None, -/// }; -/// ``` -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct QueryWorkloadEntry { - /// The query text in the workload's language. - pub query: Query, - /// Its accuracy target and latency bound (default: exact, no bound). - pub requirements: QueryRequirements, - /// How far ahead it is known: a build may run at ingestion time only for - /// `Predictable` queries (default: unknown). - pub predictability: Predictability, - /// How often it runs: sets how often each node is priced, and only - /// repeated queries may keep state from ingestion time (default: unknown). - pub recurrence: QueryRecurrence, - /// How each run is answered as a range request, one evaluation per - /// step; `None` (the default) for one evaluation per run. Stage 1 and - /// Stage 2 keep tumbling-window widths on its step grid, and Stage 3 prices - /// [`RangeEvaluation::evaluations_per_request`] evaluations per run. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub range_evaluation: Option, -} - -impl QueryWorkloadEntry { - /// The grid every evaluation of this query falls on, in milliseconds, - /// anchored at the epoch (every evaluation time is a multiple of it): - /// its fixed interval ([`QueryRecurrence::fixed_interval_ms`]), or with - /// a range evaluation of more than one step the greatest common divisor - /// of the interval and the step, since each run evaluates at every step - /// up to its end; with a declared evaluation phase - /// ([`RepeatedDemand::FixedIntervalAt`]) also the greatest common - /// divisor with that phase (every 20 s at phase 5 s is on a 5 s grid). - /// `None` when it does not repeat at a positive fixed interval. - pub fn evaluation_grid_ms(&self) -> Option { - let interval_ms = self.recurrence.fixed_interval_ms()?; - let step_ms = match self.range_evaluation { - Some(range) if range.step_ms > 0 && range.evaluations_per_request() > 1 => { - greatest_common_divisor(interval_ms, range.step_ms) - } - _ => interval_ms, - }; - let phase_ms = match &self.recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - evaluation_phase, .. - }) => evaluation_phase.0, - _ => 0, - }; - Some(greatest_common_divisor(step_ms, phase_ms)) - } -} - -impl From<&BatchEntry> for QueryWorkloadEntry { - fn from(entry: &BatchEntry) -> Self { - Self { - query: entry.query.clone(), - requirements: entry.requirements.clone(), - predictability: entry.predictability.clone(), - recurrence: QueryRecurrence::OneTime { - invocations: entry.invocations, - execute_at: entry.execute_at, - }, - range_evaluation: entry.range_evaluation, - } - } -} - -impl From<&RepeatingEntry> for QueryWorkloadEntry { - fn from(entry: &RepeatingEntry) -> Self { - Self { - query: entry.query.clone(), - requirements: entry.requirements.clone(), - predictability: entry.predictability.clone(), - recurrence: QueryRecurrence::Repeated(entry.demand.clone()), - range_evaluation: entry.range_evaluation, - } - } -} - -// ── Data workload ───────────────────────────────────────────────────────────── - -/// How the queried data arrives: already stored, still arriving, or both. -/// This is independent of whether queries repeat. -/// -/// JSON: `"at_rest"`, `{"ingesting": {"rows_per_second": 1000.0}}`, -/// `{"mixed": {"rows_per_second": 1000.0}}` or `"unknown"` (the default). -#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case", deny_unknown_fields)] -pub enum DataArrival { - /// Every row is already stored; nothing arrives, so no summary can be - /// built at ingestion time. - AtRest, - /// Rows keep arriving while the queries run. - Ingesting { - /// Rows (PromQL samples) arriving per second over all sources. - /// Plan selection prices a source's scans with its share of it, by - /// series count when every source declares one, else with all of - /// it, when the source does not declare both its `series_count` and - /// its `sample_interval` ([`SourceFacts`]). - rows_per_second: f64, - }, - /// Older rows are stored and new rows keep arriving. - Mixed { - /// Rows (PromQL samples) arriving per second over all sources, as - /// for [`DataArrival::Ingesting`]. - rows_per_second: f64, - }, - /// Not declared: no summary is built at ingestion time. - #[default] - Unknown, -} - -impl DataArrival { - /// Rows arriving per second, when the data is ingesting or mixed. - pub fn rows_per_second(&self) -> Option { - match self { - Self::Ingesting { rows_per_second } | Self::Mixed { rows_per_second } => { - Some(*rows_per_second) - } - Self::AtRest | Self::Unknown => None, - } - } -} - -impl DemandEstimate { - /// Whether this estimate was already observed and has not expired at - /// `now_ms`. A validity duration without an observation time is not a - /// usable freshness contract. - #[cfg(test)] - fn is_fresh_at(&self, now_ms: u64) -> bool { - match (self.observed_at, self.valid_for) { - (Some(observed), _) if observed.0 > now_ms => false, - (Some(observed), Some(valid_for)) => now_ms <= observed.0.saturating_add(valid_for.0), - (None, Some(_)) => false, - _ => true, - } - } -} - -/// Queries per second, samples per second, or another rate whose unit is -/// established by the field that contains it. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -#[serde(transparent)] -pub struct Rate(pub f64); - -/// Facts about the data being queried: how it arrives, and what is known -/// about each source. Every fact is declared: the planner has no clock, so a -/// caller drops facts that no longer hold before planning. -/// -/// JSON: `{"arrival": {"ingesting": {"rows_per_second": 66666.7}}, -/// "sources": {"http_requests_total": {"series_count": 1000000, -/// "metric_type": "counter", "sample_interval": 15000}}}`. -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct DataWorkload { - /// How the data arrives (default: unknown). - pub arrival: DataArrival, - /// Facts per source, keyed by PromQL metric name or SQL table name - /// (default: none). A declared histogram or summary is keyed by its - /// family name; its `metric_type`, `values_non_negative` and - /// `sample_interval` apply to every series of the family - /// ([`Self::series_facts`]). Counts, norms and bounds (`series_count`, - /// `stream_norms`, …) differ per series name (a `foo_bucket` has one - /// series per `le`), so they are read from the series' own entry; - /// pricing a source's own arrival rate reads its `series_count` and - /// `sample_interval` there together. A source without an entry has no - /// known facts. - #[serde(default)] - pub sources: BTreeMap, -} - -/// What is known about one source (a PromQL metric or a SQL table). Every -/// fact is optional; an unknown fact is never read as zero. -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SourceFacts { - /// How many series (PromQL) or groups of rows (SQL) the source holds - /// (default: unknown). Plan selection bounds the values a query returns - /// (`G`) by it, to split a whole-result failure probability, and prices - /// scans and selectors with it (with `sample_interval`, the source's - /// arrival rate). Unknown, pricing bounds a per-series count by the rows - /// read instead. Must be positive. - #[serde(default)] - pub series_count: Option, - /// Declared Prometheus type of the metric (`# TYPE` metadata). The - /// planner never infers a type from a name such as `_total` or - /// `_bucket`; an undeclared metric's samples may take any value. - #[serde(default)] - pub metric_type: Option, - /// Whether every sample of the metric is non-negative (default `false`: - /// unknown). A counter's samples always are; this declares it for a - /// gauge, such as a latency or a queue length. The deployment must - /// ensure it. Plan selection reads it as the sign of the metric's - /// samples, which a relative-error bound needs to carry through a - /// `sum` or `avg`, and as proof that Count-Min weights over the raw - /// samples are non-negative. Omitted from JSON when `false`. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub values_non_negative: bool, - /// How often each series of the metric (or group of rows of a SQL - /// table) receives a sample (default: unknown). A PromQL or MetricsQL - /// bare instant selector over the metric reads the latest sample within it; lowering such a - /// selector without it fails with - /// [`WorkloadError::MissingSampleInterval`]. With `series_count`, it - /// gives the source's own arrival rate, `series_count` samples per - /// `sample_interval`, which plan selection prices the source's scans - /// and time ranges at. - #[serde(default)] - pub sample_interval: Option, - /// Bounds on the source's stream norms: the inputs Count-Min and Count - /// Sketch additive error bounds are normalized by. Plan selection binds - /// them to turn those bounds into absolute errors; without them that - /// bound is unmet. - #[serde(default)] - pub stream_norms: Option, - /// Upper bound on the distinct items any one distinct-count summary over - /// the source's own rows counts in one complete evaluation, including - /// every merged tumbling window. One bound serves every query over the source, so - /// it must hold for the longest evaluation window any of them uses; a - /// filter or a shorter window only lowers it. Each group's own items - /// count separately: the bound is per output row. The deployment must - /// enforce it; an observed distinct count is not a bound. Plan - /// selection binds it to give classic HLL (and a distinct-count UnivMon) - /// its failure probability per output row; without it that probability - /// is unknown. KMV's guarantee needs no bound. - #[serde(default)] - pub max_distinct_per_evaluation: Option, - /// How many distinct values each label (PromQL) or column (SQL) of the - /// source takes, keyed by its name (default: none declared), e.g. - /// `{"host": 1000, "job": 10}`. Plan selection prices the groups of a - /// `by (k1, k2, ...)` at the product of its keys' counts, at most the - /// `series_count` (series or groups of rows) of the sources it reads - /// and the rows it reads per evaluation. No count is assumed for an - /// undeclared key: such a grouping is priced at the smaller of those - /// two. When every key of a - /// grouped result declares a count, their product (at most the series - /// read) also bounds the values one evaluation returns (`G`), which a - /// whole-result accuracy target splits its failure probability over: - /// each count must be an upper bound, counting a missing label (or a - /// NULL) as one more value, and positive. Omitted from JSON when empty. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub distinct_values_per_label: BTreeMap, - /// How many distinct combinations sets of labels (or columns) take - /// together, for labels that depend on each other (default: none - /// declared), e.g. `[{"labels": ["service", "endpoint"], - /// "distinct_values": 625}]` when each of 625 endpoints is in one of 25 - /// services. Plan selection prices a grouping at the smallest product - /// of declared counts whose labels cover its keys - /// (`SourceFacts::declared_distinct_combinations`), so - /// `by (service, endpoint)` is 625 groups rather than 25 x 625, and - /// `by (region, service, endpoint)` is at most 4 x 625. Nothing is - /// inferred from the per-label counts. Each set names two or more - /// distinct labels, and its count is a positive upper bound (a missing - /// label or NULL counts as one more value) no larger than the product - /// of its labels' `distinct_values_per_label` counts when all are - /// declared. Omitted from JSON when empty. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub distinct_values_per_label_set: Vec, - /// How many rows (PromQL samples) the source holds at rest, at most - /// (default: unknown). Plan selection bounds the rows a scan reads by - /// it: the whole source for a scan with no window above it, such as a - /// SQL query with no time predicate, and data at rest that declares no - /// arrival rate. Must be positive. Omitted from JSON when unknown. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub rows_held: Option, -} - -/// The distinct combinations of values a set of labels (or columns) of one -/// source takes together, an upper bound. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct LabelSetDistinctValues { - /// The labels (PromQL) or columns (SQL), two or more, in any order. - pub labels: Vec, - /// How many distinct combinations of their values the source holds. - pub distinct_values: u64, -} - -/// The most keys declared label sets are searched over: covering them -/// searches every subset of the keys some declared set names. -const MAX_KEYS_COVERED_BY_LABEL_SETS: usize = 16; - -impl SourceFacts { - /// An upper bound on the distinct combinations of values `labels` take - /// together in this source, from declared counts only: the smallest - /// product of counts, of declared label sets - /// (`distinct_values_per_label_set`) and of single labels - /// (`distinct_values_per_label`), whose labels together cover - /// `labels`. A set covering labels beyond `labels` still bounds them. - /// `Some(1)` for no labels; `None` when some label is in no declared - /// count. Only labels some declared set names are searched over (every - /// other label contributes its own count); with more than 16 of them, - /// only the per-label product is used. - pub(crate) fn declared_distinct_combinations(&self, labels: &[&str]) -> Option { - let mut keys = labels.to_vec(); - keys.sort_unstable(); - keys.dedup(); - let label_product = |keys: &[&str]| { - keys.iter().try_fold(1u64, |product, key| { - Some(product.saturating_mul(*self.distinct_values_per_label.get(*key)?)) - }) - }; - let (in_sets, alone): (Vec<&str>, Vec<&str>) = keys.iter().partition(|key| { - self.distinct_values_per_label_set - .iter() - .any(|set| set.labels.iter().any(|label| label == *key)) - }); - if in_sets.is_empty() || in_sets.len() > MAX_KEYS_COVERED_BY_LABEL_SETS { - return label_product(&keys); - } - // Each declared count over `in_sets` as (the keys it covers as a bit - // mask, count). - let mut counts: Vec<(usize, u64)> = in_sets - .iter() - .enumerate() - .filter_map(|(bit, key)| Some((1 << bit, *self.distinct_values_per_label.get(*key)?))) - .collect(); - counts.extend(self.distinct_values_per_label_set.iter().filter_map(|set| { - let covered = in_sets - .iter() - .enumerate() - .filter(|(_, key)| set.labels.iter().any(|label| label == *key)) - .fold(0, |mask, (bit, _)| mask | 1 << bit); - (covered != 0).then_some((covered, set.distinct_values)) - })); - // smallest[m]: the smallest product of counts covering the keys in m. - let every_key = (1usize << in_sets.len()) - 1; - let mut smallest: Vec> = vec![None; every_key + 1]; - smallest[0] = Some(1); - for keys_left in 1..=every_key { - smallest[keys_left] = counts - .iter() - .filter(|(covered, _)| covered & keys_left != 0) - .filter_map(|(covered, count)| { - Some(smallest[keys_left & !covered]?.saturating_mul(*count)) - }) - .min(); - } - Some(smallest[every_key]?.saturating_mul(label_product(&alone)?)) - } -} - -/// Upper bounds on the norms of the frequency (weight) vector any one -/// summary evaluation over a source folds. A filter or a shorter window only -/// lowers them, so a bound for the whole source holds for every summary -/// over it. -#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StreamNorms { - /// `‖f‖₁`, the total weight: Count-Min's bound is `ε · ‖f‖₁`. - pub l1: Option, - /// `‖f‖₂`: Count Sketch's bound is `ε · ‖f‖₂`. - pub l2: Option, -} - -/// The Prometheus type of a metric family (`# TYPE` metadata). Not an ASAP -/// summary: [`Self::Summary`] is the Prometheus summary type. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum PrometheusMetricType { - /// Starts at zero and only increases, or resets to zero: every sample is - /// non-negative. - Counter, - /// May take any value. - Gauge, - /// A classic histogram family `foo`: cumulative bucket counters - /// `foo_bucket{le=…}`, and the counters `foo_sum` and `foo_count`. The - /// planner reads `foo_sum` as a counter, so it assumes the family's - /// observations are non-negative. - Histogram, - /// A native histogram: its samples are histograms, which the IR cannot - /// represent yet, so a query reading one is rejected. - NativeHistogram, - /// A Prometheus summary family `foo`: quantiles each client computed per - /// series, `foo{quantile="φ"}`, and the counters `foo_sum` and - /// `foo_count`. - Summary, -} - -// ── Top-level workload ──────────────────────────────────────────────────────── - -/// `query_batch` and `repeating_queries` may both be present. [`Self::entries`] -/// normalizes them into one ordered stream without conflating recurrence with -/// data arrival. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct QueryWorkload { - /// Source language shared by all queries in this workload. - pub language: QueryLanguage, - /// One-shot queries executed together as a batch. - pub query_batch: Option>, - /// Queries that repeat on a fixed interval. - pub repeating_queries: Option>, -} - -/// Complete planner input. Query demand and source-data evidence are parallel -/// concerns and have independent provenance and update lifecycles. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PlanningWorkload { - pub query_workload: QueryWorkload, - pub data_workload: Option, -} - -impl QueryWorkload { - /// One normalized entry stream, independent of the source's legacy - /// batch/repeating containers. Mixed workloads preserve both kinds. - pub fn entries(&self) -> impl Iterator + '_ { - self.query_batch - .iter() - .flatten() - .map(QueryWorkloadEntry::from) - .chain( - self.repeating_queries - .iter() - .flatten() - .map(QueryWorkloadEntry::from), - ) - } - - pub(crate) fn validate(&self) -> Result<(), WorkloadError> { - for entry in self.entries() { - validate_entry(&entry)?; - } - Ok(()) - } -} - -impl DataWorkload { - /// The facts of the source a PromQL selector over `series` reads, and - /// that series' own type. A declared histogram family `foo` holds the - /// series `foo_bucket`, `foo_sum` and `foo_count`; a declared summary - /// family `foo` holds `foo`, `foo_sum` and `foo_count`; `_sum` and - /// `_count` series are counters. A declared family wins over a series' - /// own entry (`foo_sum`'s entry is ignored here when `foo` is a declared - /// histogram). Any other name is its own source, looked up as is: a - /// suffix never makes a type. - pub fn series_facts( - &self, - series: &str, - ) -> Option<(&SourceFacts, Option)> { - use PrometheusMetricType::{Counter, Histogram, Summary}; - for (suffix, series_type) in [ - ("_bucket", None), - ("_sum", Some(Counter)), - ("_count", Some(Counter)), - ] { - let Some(family) = series.strip_suffix(suffix) else { - continue; - }; - let Some(facts) = self.sources.get(family) else { - continue; - }; - match (facts.metric_type, series_type) { - (Some(Histogram), None) => return Some((facts, Some(Histogram))), - (Some(Histogram | Summary), Some(counter)) => return Some((facts, Some(counter))), - _ => {} - } - } - let facts = self.sources.get(series)?; - Some((facts, facts.metric_type)) - } - - /// Data of unknown arrival whose `metrics` are each sampled every - /// `sample_interval`, with no other facts: what PromQL or MetricsQL - /// queries over them need to lower their bare instant selectors. - pub fn sampled_every( - metrics: impl IntoIterator, - sample_interval: DurationMs, - ) -> Self { - let sampled = SourceFacts { - sample_interval: Some(sample_interval), - ..SourceFacts::default() - }; - Self { - arrival: DataArrival::Unknown, - sources: metrics - .into_iter() - .map(|metric| (metric, sampled.clone())) - .collect(), - } - } - - /// This data workload with every source it declares holding - /// `series_count` series (or groups of rows). - #[cfg(any(test, feature = "test-support"))] - pub fn with_series_count(mut self, series_count: u64) -> Self { - for facts in self.sources.values_mut() { - facts.series_count = Some(series_count); - } - self - } - - /// Validate the arrival and source facts independently of query demand - /// or language. - pub fn validate(&self) -> Result<(), WorkloadError> { - validate_optional_rate(self.arrival.rows_per_second().map(Rate))?; - for (source_name, facts) in &self.sources { - if facts.series_count == Some(0) { - return Err(WorkloadError::ZeroSeriesCount); - } - if facts.sample_interval == Some(DurationMs(0)) { - return Err(WorkloadError::ZeroSampleInterval); - } - if let Some(norms) = &facts.stream_norms { - for value in [norms.l1, norms.l2].into_iter().flatten() { - if !(value.is_finite() && value >= 0.0) { - return Err(WorkloadError::InvalidStreamNorm(value)); - } - } - } - if facts.max_distinct_per_evaluation == Some(0) { - return Err(WorkloadError::ZeroMaxDistinct); - } - if facts - .distinct_values_per_label - .values() - .any(|&count| count == 0) - { - return Err(WorkloadError::ZeroDistinctValues); - } - if facts.rows_held == Some(0) { - return Err(WorkloadError::ZeroRowsHeld); - } - for set in &facts.distinct_values_per_label_set { - let error_labels = || set.labels.clone(); - let mut distinct_labels: Vec<&String> = set.labels.iter().collect(); - distinct_labels.sort_unstable(); - distinct_labels.dedup(); - if distinct_labels.len() < 2 || distinct_labels.len() != set.labels.len() { - return Err(WorkloadError::LabelSetNeedsTwoLabels { - source_name: source_name.clone(), - labels: error_labels(), - }); - } - if set.distinct_values == 0 { - return Err(WorkloadError::ZeroDistinctValues); - } - let label_product = set.labels.iter().try_fold(1u64, |product, label| { - Some(product.saturating_mul(*facts.distinct_values_per_label.get(label)?)) - }); - if let Some(label_product) = label_product.filter(|&p| set.distinct_values > p) { - return Err(WorkloadError::LabelSetCountAboveLabelProduct { - source_name: source_name.clone(), - labels: error_labels(), - distinct_values: set.distinct_values, - label_product, - }); - } - } - } - Ok(()) - } -} - -impl PlanningWorkload { - /// Validate the query workload and, when present, the data workload. A - /// PromQL or MetricsQL sample interval is checked by the frontend, only - /// for a query with a bare instant selector - /// ([`WorkloadError::MissingSampleInterval`]). - pub fn validate(&self) -> Result<(), WorkloadError> { - self.query_workload.validate()?; - if let Some(data) = &self.data_workload { - data.validate()?; - } - Ok(()) - } - - /// The sample interval the data workload declares for the source - /// `series` reads ([`DataWorkload::series_facts`]), if any. - pub fn sample_interval_of(&self, series: &str) -> Option { - self.data_workload - .as_ref()? - .series_facts(series)? - .0 - .sample_interval - } -} - -fn validate_entry(entry: &QueryWorkloadEntry) -> Result<(), WorkloadError> { - if let Some(ms) = entry.requirements.max_latency_ms { - if !ms.is_finite() || ms < 0.0 { - return Err(WorkloadError::InvalidLatency(ms)); - } - } - match &entry.recurrence { - QueryRecurrence::OneTime { invocations: 0, .. } => { - return Err(WorkloadError::ZeroInvocations) - } - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(0))) - | QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(0), - .. - }) => return Err(WorkloadError::ZeroRepetitionInterval), - QueryRecurrence::Repeated(RepeatedDemand::Scheduled(schedule)) if schedule.is_empty() => { - return Err(WorkloadError::EmptySchedule) - } - QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { - validate_rate(estimate.expected_rate)?; - } - _ => {} - } - if let Some(range) = entry.range_evaluation { - if range.step_ms == 0 { - return Err(WorkloadError::ZeroRangeEvaluationStep); - } - if range.span_ms % range.step_ms != 0 { - return Err(WorkloadError::RangeEvaluationSpanNotMultipleOfStep { - span_ms: range.span_ms, - step_ms: range.step_ms, - }); - } - let steps = range.evaluations_per_request(); - if steps > MAX_RANGE_EVALUATION_STEPS { - return Err(WorkloadError::TooManyRangeEvaluationSteps { steps }); - } - } - Ok(()) -} - -fn validate_optional_rate(rate: Option) -> Result<(), WorkloadError> { - rate.map(validate_rate).transpose().map(|_| ()) -} - -fn validate_rate(rate: Rate) -> Result { - if rate.0.is_finite() && rate.0 >= 0.0 { - Ok(rate) - } else { - Err(WorkloadError::InvalidRate(rate.0)) - } -} - -/// Why a planning workload is rejected. Every variant is bad input. -#[derive(Debug, Clone, PartialEq, thiserror::Error)] -pub enum WorkloadError { - /// Bad input: a PromQL or MetricsQL query has a bare instant selector, - /// which reads the latest sample within its metric's sample interval, - /// but the data workload declares no `sample_interval` for that metric. - #[error( - "a PromQL or MetricsQL instant selector needs its metric's sample_interval \ - in data_workload.sources" - )] - MissingSampleInterval, - /// Bad input: a source declares a sample interval of zero. - #[error("sample_interval must be greater than zero")] - ZeroSampleInterval, - /// Bad input: a one-time query with zero invocations. - #[error("a one-time query must have at least one invocation")] - ZeroInvocations, - /// Bad input: a repeating query whose fixed interval is zero. - #[error("a fixed repetition interval must be greater than zero")] - ZeroRepetitionInterval, - /// Bad input: a range evaluation whose `step_ms` is zero. - #[error("a range evaluation's step_ms must be greater than zero")] - ZeroRangeEvaluationStep, - /// Bad input: a range evaluation whose `span_ms` is not a whole number - /// of steps. Prometheus places steps forward from the start, so its - /// last step would not be the request's end and would fall off the - /// end's grid. - #[error( - "a range evaluation's span_ms ({span_ms}) must be a multiple of its step_ms ({step_ms})" - )] - RangeEvaluationSpanNotMultipleOfStep { - /// The declared span. - span_ms: u64, - /// The declared step. - step_ms: u64, - }, - /// Bad input: a range evaluation with more steps per request than - /// `MAX_RANGE_EVALUATION_STEPS`, which Prometheus would reject. - #[error( - "a range evaluation has {steps} steps per request, more than {}", - MAX_RANGE_EVALUATION_STEPS - )] - TooManyRangeEvaluationSteps { - /// The steps one request would evaluate, `span_ms / step_ms + 1`. - steps: u64, - }, - /// Bad input: a scheduled repeating query with no timestamps. - #[error("a repeated-query schedule must not be empty")] - EmptySchedule, - /// Bad input: a rate (a demand estimate's, or the arrival's - /// `rows_per_second`) that is negative or not finite. - #[error("rate must be finite and non-negative, got {0}")] - InvalidRate(f64), - /// Bad input: a query's `max_latency_ms` that is negative or not - /// finite. - #[error("response latency must be finite and non-negative, got {0} ms")] - InvalidLatency(f64), - /// Bad input: a source's stream norm bound that is negative or not - /// finite. - #[error("a stream norm must be finite and non-negative, got {0}")] - InvalidStreamNorm(f64), - /// Bad input: a source's `max_distinct_per_evaluation` of zero. - #[error("a source's max_distinct_per_evaluation must be greater than zero")] - ZeroMaxDistinct, - /// Bad input: a zero count in a source's `distinct_values_per_label` - /// or `distinct_values_per_label_set`. - #[error( - "a source's distinct_values_per_label and distinct_values_per_label_set counts must be \ - greater than zero" - )] - ZeroDistinctValues, - /// Bad input: a `distinct_values_per_label_set` entry naming fewer than - /// two labels, or one label twice (one label's count belongs in - /// `distinct_values_per_label`). - #[error( - "source {source_name}: a distinct_values_per_label_set entry must name two or more \ - distinct labels, got {labels:?}" - )] - LabelSetNeedsTwoLabels { - source_name: String, - labels: Vec, - }, - /// Bad input: a label set declaring more combinations than the product - /// of its labels' declared distinct values allows. - #[error( - "source {source_name}: distinct_values_per_label_set declares {distinct_values} \ - combinations of {labels:?}, more than the {label_product} their \ - distinct_values_per_label counts allow" - )] - LabelSetCountAboveLabelProduct { - source_name: String, - labels: Vec, - distinct_values: u64, - label_product: u64, - }, - /// Bad input: a source's `rows_held` of zero. - #[error("a source's rows_held must be greater than zero")] - ZeroRowsHeld, - /// Bad input: a source declares a `series_count` of zero. A source - /// with no series has nothing to plan for, and `absent(m)` over it - /// still returns one row. - #[error("a source's series_count must be greater than zero")] - ZeroSeriesCount, -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Only a fixed-interval repetition with a positive interval has a - /// fixed interval; a zero interval, a schedule, an estimated rate, a - /// one-time query and an unknown recurrence have none. - #[test] - fn fixed_interval_ms_is_set_only_for_a_positive_fixed_interval() { - let repeated = |demand| QueryRecurrence::Repeated(demand); - assert_eq!( - repeated(RepeatedDemand::FixedInterval(RepetitionInterval(10_000))).fixed_interval_ms(), - Some(10_000) - ); - assert_eq!( - repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(60_000), - evaluation_phase: TimestampMs(5_000), - }) - .fixed_interval_ms(), - Some(60_000) - ); - assert_eq!( - repeated(RepeatedDemand::FixedInterval(RepetitionInterval(0))).fixed_interval_ms(), - None - ); - assert_eq!( - repeated(RepeatedDemand::Scheduled(vec![TimestampMs(1_000)])).fixed_interval_ms(), - None - ); - assert_eq!( - QueryRecurrence::OneTime { - invocations: 1, - execute_at: None, - } - .fixed_interval_ms(), - None - ); - assert_eq!(QueryRecurrence::Unknown.fixed_interval_ms(), None); - } - - /// Query requirements: an accuracy target (omitted: exact) and an - /// optional latency bound in ms (omitted: none). The old tagged forms - /// and unknown fields are rejected. - #[test] - fn query_requirements_json_is_a_target_and_an_optional_bound() { - let parse = |json: &str| serde_json::from_str::(json); - let empty = serde_json::to_value(parse("{}").unwrap()).unwrap(); - assert_eq!( - empty, - serde_json::json!({"accuracy": {"max_error": 0.0, "failure_probability": 0.0, - "accuracy_scope": "whole_result"}, "max_latency_ms": null}) - ); - let bounded = serde_json::to_value( - parse(r#"{"accuracy": {"max_error": 0.01}, "max_latency_ms": 200.0}"#).unwrap(), - ) - .unwrap(); - assert_eq!(bounded["accuracy"]["max_error"], 0.01); - assert_eq!(bounded["max_latency_ms"], 200.0); - for old in [ - r#"{"accuracy": "implicit_exact", "response_latency": "unspecified"}"#, - r#"{"accuracy": {"explicit": {"max_error": 0.01}}}"#, - r#"{"response_latency": {"explicit_max_ms": 100.0}}"#, - r#"{"max_latency_ms": 100.0, "unknown": 1}"#, - ] { - assert!(parse(old).is_err(), "{old}"); - } - } - - /// A stream norm is a finite, non-negative magnitude. - #[test] - fn stream_norms_must_be_finite_and_non_negative() { - let with = |l1: f64| DataWorkload { - sources: BTreeMap::from([( - "m".to_string(), - SourceFacts { - stream_norms: Some(StreamNorms { - l1: Some(l1), - l2: None, - }), - ..SourceFacts::default() - }, - )]), - ..DataWorkload::default() - }; - assert!(with(10.0).validate().is_ok()); - for bad in [-1.0, f64::NAN, f64::INFINITY] { - assert!( - matches!( - with(bad).validate(), - Err(WorkloadError::InvalidStreamNorm(_)) - ), - "{bad}" - ); - } - } - - /// A distinct-count bound is positive: a summary counts at least one item. - #[test] - fn max_distinct_per_evaluation_must_be_positive() { - let with = |bound: u64| DataWorkload { - sources: BTreeMap::from([( - "m".to_string(), - SourceFacts { - max_distinct_per_evaluation: Some(bound), - ..SourceFacts::default() - }, - )]), - ..DataWorkload::default() - }; - assert!(with(1).validate().is_ok()); - assert_eq!(with(0).validate(), Err(WorkloadError::ZeroMaxDistinct)); - } - - /// A declared `rows_held` is positive. - #[test] - fn rows_held_must_be_positive() { - let with = |rows: u64| DataWorkload { - sources: BTreeMap::from([( - "t".to_string(), - SourceFacts { - rows_held: Some(rows), - ..SourceFacts::default() - }, - )]), - ..DataWorkload::default() - }; - assert!(with(1).validate().is_ok()); - assert_eq!(with(0).validate(), Err(WorkloadError::ZeroRowsHeld)); - } - - /// A declared distinct-value count is positive: a label with no value - /// would make every grouping over it empty. - #[test] - fn distinct_values_per_label_must_be_positive() { - let with = |count: u64| DataWorkload { - sources: BTreeMap::from([( - "m".to_string(), - SourceFacts { - distinct_values_per_label: BTreeMap::from([ - ("host".to_string(), 1_000), - ("job".to_string(), count), - ]), - ..SourceFacts::default() - }, - )]), - ..DataWorkload::default() - }; - assert!(with(1).validate().is_ok()); - assert_eq!(with(0).validate(), Err(WorkloadError::ZeroDistinctValues)); - } - - /// The `http` source of the eval's RollupHeavy workload: 4 regions, 25 - /// services and 625 endpoints, each endpoint in one service, so - /// (service, endpoint) takes 625 combinations, not 25 x 625. - fn hierarchical_facts(declared_sets: &[(&[&str], u64)]) -> SourceFacts { - SourceFacts { - distinct_values_per_label: BTreeMap::from([ - ("region".to_string(), 4), - ("service".to_string(), 25), - ("endpoint".to_string(), 625), - ]), - distinct_values_per_label_set: declared_sets - .iter() - .map(|(labels, distinct_values)| LabelSetDistinctValues { - labels: labels.iter().map(|label| label.to_string()).collect(), - distinct_values: *distinct_values, - }) - .collect(), - ..SourceFacts::default() - } - } - - /// A grouping's combinations are bounded by the smallest product over - /// declared sets and labels covering its keys: the declared set itself, - /// a superset of keys through the set times the other labels, and a - /// subset of a declared set through that set. Without a declared set, - /// the per-label product stands. - #[test] - fn declared_label_sets_bound_the_combinations_of_their_labels() { - let declared = hierarchical_facts(&[(&["service", "endpoint"], 625)]); - let undeclared = hierarchical_facts(&[]); - let combinations = - |facts: &SourceFacts, labels: &[&str]| facts.declared_distinct_combinations(labels); - assert_eq!(combinations(&declared, &["service", "endpoint"]), Some(625)); - assert_eq!(combinations(&declared, &["endpoint", "service"]), Some(625)); - assert_eq!( - combinations(&declared, &["region", "service", "endpoint"]), - Some(4 * 625) - ); - assert_eq!( - combinations(&undeclared, &["service", "endpoint"]), - Some(25 * 625) - ); - assert_eq!( - combinations(&undeclared, &["region", "service", "endpoint"]), - Some(4 * 25 * 625) - ); - assert_eq!(combinations(&declared, &["service"]), Some(25)); - assert_eq!(combinations(&declared, &[]), Some(1)); - // A key no fact covers leaves the combinations unbound. - assert_eq!(combinations(&declared, &["service", "status"]), None); - // A declared set covering more labels bounds its subsets: 3 users - // have made 7 (user, session) pairs, so at most 7 sessions, while - // `session` alone declares no count. - let sessions = SourceFacts { - distinct_values_per_label_set: vec![LabelSetDistinctValues { - labels: vec!["user".to_string(), "session".to_string()], - distinct_values: 7, - }], - ..SourceFacts::default() - }; - assert_eq!( - sessions.declared_distinct_combinations(&["session"]), - Some(7) - ); - // 70 keys, two in a declared set: only those two are searched over. - let mut wide = hierarchical_facts(&[(&["service", "endpoint"], 625)]); - let names: Vec = (0..68).map(|i| format!("l{i}")).collect(); - for name in &names { - wide.distinct_values_per_label.insert(name.clone(), 1); - } - let mut labels: Vec<&str> = names.iter().map(String::as_str).collect(); - labels.extend(["service", "endpoint"]); - assert_eq!(wide.declared_distinct_combinations(&labels), Some(625)); - } - - /// A declared set count is an input error, naming the source, when it - /// is above the product of its labels' declared counts, or when the set - /// has fewer than two distinct labels (one label's count belongs in - /// `distinct_values_per_label`) or zero values. - #[test] - fn a_label_set_count_is_at_most_its_labels_product() { - let with = |facts: SourceFacts| DataWorkload { - sources: BTreeMap::from([("http".to_string(), facts)]), - ..DataWorkload::default() - }; - assert!(with(hierarchical_facts(&[(&["service", "endpoint"], 625)])) - .validate() - .is_ok()); - // `status` declares no count, so the set's product is unknown. - assert!(with(hierarchical_facts(&[(&["service", "status"], 1_000)])) - .validate() - .is_ok()); - let error = with(hierarchical_facts(&[(&["region", "service"], 101)])) - .validate() - .unwrap_err(); - assert_eq!( - error, - WorkloadError::LabelSetCountAboveLabelProduct { - source_name: "http".to_string(), - labels: vec!["region".to_string(), "service".to_string()], - distinct_values: 101, - label_product: 100, - } - ); - assert!(error.to_string().contains("http"), "{error}"); - for labels in [&["service"][..], &["service", "service"][..]] { - let error = with(hierarchical_facts(&[(labels, 25)])).validate(); - assert!( - matches!( - &error, - Err(WorkloadError::LabelSetNeedsTwoLabels { source_name, .. }) - if source_name == "http" - ), - "{error:?}" - ); - } - assert_eq!( - with(hierarchical_facts(&[(&["service", "endpoint"], 0)])).validate(), - Err(WorkloadError::ZeroDistinctValues) - ); - } - - /// A declared series count is positive: with zero series, `absent(m)` - /// would still return one row the bound does not count. - #[test] - fn series_count_must_be_positive() { - let with = |series_count: u64| DataWorkload { - sources: BTreeMap::from([( - "m".to_string(), - SourceFacts { - series_count: Some(series_count), - ..SourceFacts::default() - }, - )]), - ..DataWorkload::default() - }; - assert!(with(1).validate().is_ok()); - assert_eq!(with(0).validate(), Err(WorkloadError::ZeroSeriesCount)); - } - - /// The arrival's rate is finite and non-negative, and each source's - /// declared sample interval is positive. - #[test] - fn arrival_rate_and_sample_interval_are_validated() { - let with = |rows_per_second: f64, sample_interval: Option| DataWorkload { - arrival: DataArrival::Mixed { rows_per_second }, - sources: BTreeMap::from([( - "m".to_string(), - SourceFacts { - sample_interval: sample_interval.map(DurationMs), - ..SourceFacts::default() - }, - )]), - }; - assert!(with(10.0, Some(15_000)).validate().is_ok()); - assert!(with(0.0, None).validate().is_ok()); - assert_eq!( - with(-1.0, None).validate(), - Err(WorkloadError::InvalidRate(-1.0)) - ); - for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { - assert!( - matches!( - with(bad, None).validate(), - Err(WorkloadError::InvalidRate(_)) - ), - "{bad}" - ); - } - assert_eq!( - with(1.0, Some(0)).validate(), - Err(WorkloadError::ZeroSampleInterval) - ); - } - - /// A repeating query with a zero fixed interval is rejected. - #[test] - fn zero_repetition_interval_is_rejected() { - let mut workload = base_workload(); - workload.repeating_queries = Some(vec![RepeatingEntry { - query: Query("dashboard".into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(0)), - requirements: QueryRequirements::default(), - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - }]); - assert_eq!( - workload.validate(), - Err(WorkloadError::ZeroRepetitionInterval) - ); - } - - /// A range evaluation is optional JSON on a repeating entry, carried to - /// its normalized entry; its step must be positive and a request may - /// have at most `MAX_RANGE_EVALUATION_STEPS` steps. An entry without - /// one serializes as before, with no `range_evaluation` key. - #[test] - fn range_evaluation_is_optional_carried_and_validated() { - let dashboard = |range: serde_json::Value| { - let mut entry = serde_json::json!({ - "query": "rate(requests_total[5m])", - "demand": {"fixed_interval": 30000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}, - }); - if !range.is_null() { - entry["range_evaluation"] = range; - } - serde_json::from_value::(entry).unwrap() - }; - let plain = dashboard(serde_json::Value::Null); - assert_eq!(plain.range_evaluation, None); - assert!(serde_json::to_value(&plain) - .unwrap() - .get("range_evaluation") - .is_none()); - - let hour = dashboard(serde_json::json!({"step_ms": 60000, "span_ms": 3600000})); - let range = RangeEvaluation { - step_ms: 60_000, - span_ms: 3_600_000, - }; - assert_eq!(hour.range_evaluation, Some(range)); - assert_eq!(range.evaluations_per_request(), 61); - let entry = QueryWorkloadEntry::from(&hour); - assert_eq!(entry.range_evaluation, Some(range)); - // Refreshes every 30 s and steps every 60 s: evaluations on a 30 s grid. - assert_eq!(entry.evaluation_grid_ms(), Some(30_000)); - assert_eq!( - QueryWorkloadEntry::from(&plain).evaluation_grid_ms(), - Some(30_000) - ); - // At a declared phase, also on that phase from the epoch: every - // minute at phase 125 s is on a 5 s grid; at phase 120 s, on 60 s. - let at_phase = |phase_ms| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(60_000), - evaluation_phase: TimestampMs(phase_ms), - }), - range_evaluation: None, - ..entry.clone() - }; - assert_eq!(at_phase(125_000).evaluation_grid_ms(), Some(5_000)); - assert_eq!(at_phase(120_000).evaluation_grid_ms(), Some(60_000)); - // A one-step range (span 0) is one evaluation per request: its - // step does not narrow the grid (refreshing every 60 s at a 45 s - // step would otherwise give a 15 s grid for nothing). - let one_step = QueryWorkloadEntry { - range_evaluation: Some(RangeEvaluation { - step_ms: 45_000, - span_ms: 0, - }), - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - RepetitionInterval(60_000), - )), - ..entry.clone() - }; - assert_eq!(one_step.evaluation_grid_ms(), Some(60_000)); - - let validate = |range: RangeEvaluation| { - let mut workload = base_workload(); - workload.repeating_queries = Some(vec![RepeatingEntry { - range_evaluation: Some(range), - ..hour.clone() - }]); - workload.validate() - }; - assert_eq!(validate(range), Ok(())); - assert_eq!( - validate(RangeEvaluation { - step_ms: 0, - span_ms: 60_000 - }), - Err(WorkloadError::ZeroRangeEvaluationStep) - ); - assert_eq!( - validate(RangeEvaluation { - step_ms: 1_000, - span_ms: 11_000_000 - }), - Err(WorkloadError::TooManyRangeEvaluationSteps { steps: 11_001 }) - ); - assert_eq!( - validate(RangeEvaluation { - step_ms: 1_000, - span_ms: 10_999_000 - }), - Ok(()) - ); - // Prometheus places steps forward from the start, so a span that is - // not a whole number of steps would put the last step off the end's - // grid: refreshing every 60 s at 60 s steps over 90 s evaluates at - // E − 90 s and E − 30 s. It is rejected. - assert_eq!( - validate(RangeEvaluation { - step_ms: 60_000, - span_ms: 90_000 - }), - Err(WorkloadError::RangeEvaluationSpanNotMultipleOfStep { - span_ms: 90_000, - step_ms: 60_000 - }) - ); - let unknown = serde_json::json!({"step_ms": 1, "span_ms": 0, "start": 0}); - assert!(serde_json::from_value::(unknown).is_err()); - } - - fn base_workload() -> QueryWorkload { - QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: None, - } - } - - #[test] - fn entries_keep_each_entrys_recurrence() { - let mut workload = base_workload(); - workload.query_batch = Some(vec![BatchEntry { - query: Query("historical".into()), - requirements: QueryRequirements::default(), - predictability: Predictability::AdHoc, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]); - workload.repeating_queries = Some(vec![RepeatingEntry { - query: Query("dashboard".into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), - requirements: QueryRequirements::default(), - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - }]); - - let entries: Vec<_> = workload.entries().collect(); - assert_eq!(entries.len(), 2); - assert!(matches!( - entries[0].recurrence, - QueryRecurrence::OneTime { .. } - )); - assert!(matches!( - entries[1].recurrence, - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(_)) - )); - workload.validate().unwrap(); - } - - #[test] - fn future_demand_estimates_are_not_fresh() { - let estimate = DemandEstimate { - expected_rate: Rate(1.0), - observed_at: Some(TimestampMs(2_000)), - valid_for: Some(DurationMs(1_000)), - }; - assert!(!estimate.is_fresh_at(1_999)); - assert!(estimate.is_fresh_at(2_000)); - } - - #[test] - fn canonical_workloads_have_a_strict_json_round_trip() { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query("rate(requests_total[5m])".into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget::max_error(0.01).failure_probability(0.001), - max_latency_ms: Some(100.0), - }, - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - }]), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000.0, - }, - sources: BTreeMap::from([( - "requests_total".to_string(), - SourceFacts { - series_count: Some(10_000), - metric_type: Some(PrometheusMetricType::Counter), - values_non_negative: true, - sample_interval: Some(DurationMs(15_000)), - stream_norms: Some(StreamNorms { - l1: Some(1e9), - l2: None, - }), - max_distinct_per_evaluation: Some(100), - distinct_values_per_label: BTreeMap::from([ - ("job".to_string(), 10), - ("instance".to_string(), 1_000), - ]), - distinct_values_per_label_set: vec![LabelSetDistinctValues { - labels: vec!["job".to_string(), "instance".to_string()], - distinct_values: 1_000, - }], - rows_held: Some(1_000_000), - }, - )]), - }), - }; - - let json = serde_json::to_value(&workload).expect("serialize workload"); - assert_eq!( - json["data_workload"], - serde_json::json!({ - "arrival": {"ingesting": {"rows_per_second": 1000.0}}, - "sources": {"requests_total": { - "series_count": 10000, - "metric_type": "counter", - "values_non_negative": true, - "sample_interval": 15000, - "stream_norms": {"l1": 1e9, "l2": null}, - "max_distinct_per_evaluation": 100, - "distinct_values_per_label": {"instance": 1000, "job": 10}, - "distinct_values_per_label_set": [ - {"labels": ["job", "instance"], "distinct_values": 1000} - ], - "rows_held": 1000000 - }} - }) - ); - let decoded: PlanningWorkload = serde_json::from_value(json.clone()).unwrap(); - assert_eq!(decoded, workload); - - let text = serde_json::to_string(&workload).unwrap(); - let with_unknown = text.replacen("{", "{\"unknown\":true,", 1); - assert!(serde_json::from_str::(&with_unknown).is_err()); - - // Omitted facts default to unknown. - let minimal: DataWorkload = serde_json::from_value(serde_json::json!({ - "arrival": {"mixed": {"rows_per_second": 5.0}}, - "sources": {"flows": {"series_count": 7}} - })) - .unwrap(); - assert_eq!( - minimal.sources["flows"], - SourceFacts { - series_count: Some(7), - ..SourceFacts::default() - } - ); - } - - /// The former flat shape (per-field evidence, one global cardinality, - /// per-fact source maps, `continuously_ingesting`) is rejected, as are - /// freshness evidence on a source and a sample interval on the arrival. - #[test] - fn the_former_data_workload_shape_is_rejected() { - let declared = |value: serde_json::Value| { - serde_json::json!({"value": value, "source": "declared", - "observed_at_ms": null, "valid_for_ms": null}) - }; - for old in [ - serde_json::json!({ - "arrival": "continuously_ingesting", - "data_ingestion_interval": declared(15000.into()), - "ingestion_rate": declared(1000.0.into()), - "input_cardinality": declared(10000.into()), - }), - serde_json::json!({"arrival": "continuously_ingesting"}), - serde_json::json!({"arrival": "at_rest", "input_cardinality": declared(1.into())}), - serde_json::json!({"arrival": "at_rest", - "metric_types": {"m": "counter"}}), - serde_json::json!({"arrival": "at_rest", - "sources": {"m": {"series_count": 1, "evidence": {"observed_at_ms": 1}}}}), - serde_json::json!({"arrival": "at_rest", "sources": {"m": {"series": 1}}}), - serde_json::json!({"arrival": {"ingesting": - {"rows_per_second": 1.0, "sample_interval": 15000}}}), - serde_json::json!({"arrival": {"at_rest": {"rows_per_second": 1.0}}}), - ] { - assert!( - serde_json::from_value::(old.clone()).is_err(), - "{old}" - ); - } - } - - /// `metric_type` takes each Prometheus type by its snake_case name. - #[test] - fn prometheus_metric_types_have_snake_case_wire_names() { - for (metric_type, name) in [ - (PrometheusMetricType::Counter, "counter"), - (PrometheusMetricType::Gauge, "gauge"), - (PrometheusMetricType::Histogram, "histogram"), - (PrometheusMetricType::NativeHistogram, "native_histogram"), - (PrometheusMetricType::Summary, "summary"), - ] { - let facts: SourceFacts = - serde_json::from_value(serde_json::json!({ "metric_type": name })).unwrap(); - assert_eq!(facts.metric_type, Some(metric_type)); - } - } - - /// A declared histogram family supplies its `_bucket`, `_sum` and - /// `_count` series, a declared summary family its `_sum` and `_count` - /// series, with `_sum` and `_count` counters; any other name, including - /// a suffixed one whose family is undeclared or a counter, is looked up - /// as is. - #[test] - fn series_facts_resolve_declared_histogram_and_summary_families() { - use PrometheusMetricType::{Counter, Gauge, Histogram, Summary}; - let facts = |metric_type| SourceFacts { - metric_type: Some(metric_type), - sample_interval: Some(DurationMs(15_000)), - ..SourceFacts::default() - }; - let data = DataWorkload { - arrival: DataArrival::Unknown, - sources: BTreeMap::from([ - ("h".to_string(), facts(Histogram)), - ("s".to_string(), facts(Summary)), - ("c".to_string(), facts(Counter)), - ("g_sum".to_string(), facts(Gauge)), - ]), - }; - let series_type = |series: &str| data.series_facts(series).map(|(_, t)| t); - assert_eq!(series_type("h"), Some(Some(Histogram))); - assert_eq!(series_type("h_bucket"), Some(Some(Histogram))); - assert_eq!(series_type("h_sum"), Some(Some(Counter))); - assert_eq!(series_type("h_count"), Some(Some(Counter))); - assert_eq!(series_type("s"), Some(Some(Summary))); - assert_eq!(series_type("s_sum"), Some(Some(Counter))); - assert_eq!(series_type("s_count"), Some(Some(Counter))); - assert_eq!(series_type("s_bucket"), None); - assert_eq!(series_type("c_sum"), None); - assert_eq!(series_type("g_sum"), Some(Some(Gauge))); - assert_eq!( - data.series_facts("h_sum").unwrap().0.sample_interval, - Some(DurationMs(15_000)) - ); - } - - #[test] - fn query_language_wire_names_do_not_split_acronyms() { - let cases = [ - (QueryLanguage::PromQL, serde_json::json!("promql")), - (QueryLanguage::MetricsQL, serde_json::json!("metricsql")), - ( - QueryLanguage::SQL(SqlDialect::DataFusionSQL), - serde_json::json!({"sql": "datafusion_sql"}), - ), - ( - QueryLanguage::SQL(SqlDialect::ClickhouseSQL), - serde_json::json!({"sql": "clickhouse_sql"}), - ), - (QueryLanguage::ElasticDSL, serde_json::json!("elastic_dsl")), - ]; - for (language, expected) in cases { - assert_eq!(serde_json::to_value(language).unwrap(), expected); - } - } - - #[test] - fn struct_enum_variants_reject_unknown_fields() { - assert!( - serde_json::from_value::(serde_json::json!({ - "one_time": {"invocations": 1, "execute_at": null, "typo": true} - })) - .is_err() - ); - assert!(serde_json::from_value::(serde_json::json!({ - "predictable": {"known_at": null, "typo": true} - })) - .is_err()); - assert!(serde_json::from_value::(serde_json::json!({ - "max_error": 0.1, "failure_probability": 0.01, "typo": true - })) - .is_err()); - } - - #[test] - fn estimated_demand_validates_its_rate() { - let mut workload = base_workload(); - workload.repeating_queries = Some(vec![RepeatingEntry { - query: Query("estimated".into()), - demand: RepeatedDemand::EstimatedRate(DemandEstimate { - expected_rate: Rate(-1.0), - observed_at: None, - valid_for: None, - }), - requirements: QueryRequirements::default(), - predictability: Predictability::Unknown, - range_evaluation: None, - }]); - assert_eq!(workload.validate(), Err(WorkloadError::InvalidRate(-1.0))); - } -} +mod planning; +mod query; + +pub use data::{ + DataArrival, DataWorkload, LabelSetDistinctValues, PrometheusMetricType, Rate, SourceFacts, + StreamNorms, +}; +pub use planning::{PlanningWorkload, WorkloadError}; +pub use query::{ + greatest_common_divisor, BatchEntry, DemandEstimate, DurationMs, Predictability, Query, + QueryLanguage, QueryRecurrence, QueryRequirements, QueryWorkload, QueryWorkloadEntry, + RangeEvaluation, RepeatedDemand, RepeatingEntry, RepetitionInterval, SqlDialect, TimestampMs, +}; diff --git a/crates/types/src/workload/planning.rs b/crates/types/src/workload/planning.rs new file mode 100644 index 000000000..0ed9a921a --- /dev/null +++ b/crates/types/src/workload/planning.rs @@ -0,0 +1,248 @@ +//! The planning workload: the query and data workloads planned together, +//! and why a workload is invalid. + +use serde::{Deserialize, Serialize}; + +use super::query::MAX_RANGE_EVALUATION_STEPS; +use super::{DataWorkload, DurationMs, QueryWorkload}; + +/// Complete planner input. Query demand and source-data evidence are parallel +/// concerns and have independent provenance and update lifecycles. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PlanningWorkload { + pub query_workload: QueryWorkload, + pub data_workload: Option, +} + +impl PlanningWorkload { + /// Validate the query workload and, when present, the data workload. A + /// PromQL or MetricsQL sample interval is checked by the frontend, only + /// for a query with a bare instant selector + /// ([`WorkloadError::MissingSampleInterval`]). + pub fn validate(&self) -> Result<(), WorkloadError> { + self.query_workload.validate()?; + if let Some(data) = &self.data_workload { + data.validate()?; + } + Ok(()) + } + + /// The sample interval the data workload declares for the source + /// `series` reads ([`DataWorkload::series_facts`]), if any. + pub fn sample_interval_of(&self, series: &str) -> Option { + self.data_workload + .as_ref()? + .series_facts(series)? + .0 + .sample_interval + } +} + +/// Why a planning workload is rejected. Every variant is bad input. +#[derive(Debug, Clone, PartialEq, thiserror::Error)] +pub enum WorkloadError { + /// Bad input: a PromQL or MetricsQL query has a bare instant selector, + /// which reads the latest sample within its metric's sample interval, + /// but the data workload declares no `sample_interval` for that metric. + #[error( + "a PromQL or MetricsQL instant selector needs its metric's sample_interval \ + in data_workload.sources" + )] + MissingSampleInterval, + /// Bad input: a source declares a sample interval of zero. + #[error("sample_interval must be greater than zero")] + ZeroSampleInterval, + /// Bad input: a one-time query with zero invocations. + #[error("a one-time query must have at least one invocation")] + ZeroInvocations, + /// Bad input: a repeating query whose fixed interval is zero. + #[error("a fixed repetition interval must be greater than zero")] + ZeroRepetitionInterval, + /// Bad input: a range evaluation whose `step_ms` is zero. + #[error("a range evaluation's step_ms must be greater than zero")] + ZeroRangeEvaluationStep, + /// Bad input: a range evaluation whose `span_ms` is not a whole number + /// of steps. Prometheus places steps forward from the start, so its + /// last step would not be the request's end and would fall off the + /// end's grid. + #[error( + "a range evaluation's span_ms ({span_ms}) must be a multiple of its step_ms ({step_ms})" + )] + RangeEvaluationSpanNotMultipleOfStep { + /// The declared span. + span_ms: u64, + /// The declared step. + step_ms: u64, + }, + /// Bad input: a range evaluation with more steps per request than + /// `MAX_RANGE_EVALUATION_STEPS`, which Prometheus would reject. + #[error( + "a range evaluation has {steps} steps per request, more than {}", + MAX_RANGE_EVALUATION_STEPS + )] + TooManyRangeEvaluationSteps { + /// The steps one request would evaluate, `span_ms / step_ms + 1`. + steps: u64, + }, + /// Bad input: a scheduled repeating query with no timestamps. + #[error("a repeated-query schedule must not be empty")] + EmptySchedule, + /// Bad input: a rate (a demand estimate's, or the arrival's + /// `rows_per_second`) that is negative or not finite. + #[error("rate must be finite and non-negative, got {0}")] + InvalidRate(f64), + /// Bad input: a query's `max_latency_ms` that is negative or not + /// finite. + #[error("response latency must be finite and non-negative, got {0} ms")] + InvalidLatency(f64), + /// Bad input: a source's stream norm bound that is negative or not + /// finite. + #[error("a stream norm must be finite and non-negative, got {0}")] + InvalidStreamNorm(f64), + /// Bad input: a source's `max_distinct_per_evaluation` of zero. + #[error("a source's max_distinct_per_evaluation must be greater than zero")] + ZeroMaxDistinct, + /// Bad input: a zero count in a source's `distinct_values_per_label` + /// or `distinct_values_per_label_set`. + #[error( + "a source's distinct_values_per_label and distinct_values_per_label_set counts must be \ + greater than zero" + )] + ZeroDistinctValues, + /// Bad input: a `distinct_values_per_label_set` entry naming fewer than + /// two labels, or one label twice (one label's count belongs in + /// `distinct_values_per_label`). + #[error( + "source {source_name}: a distinct_values_per_label_set entry must name two or more \ + distinct labels, got {labels:?}" + )] + LabelSetNeedsTwoLabels { + source_name: String, + labels: Vec, + }, + /// Bad input: a label set declaring more combinations than the product + /// of its labels' declared distinct values allows. + #[error( + "source {source_name}: distinct_values_per_label_set declares {distinct_values} \ + combinations of {labels:?}, more than the {label_product} their \ + distinct_values_per_label counts allow" + )] + LabelSetCountAboveLabelProduct { + source_name: String, + labels: Vec, + distinct_values: u64, + label_product: u64, + }, + /// Bad input: a source's `rows_held` of zero. + #[error("a source's rows_held must be greater than zero")] + ZeroRowsHeld, + /// Bad input: a source declares a `series_count` of zero. A source + /// with no series has nothing to plan for, and `absent(m)` over it + /// still returns one row. + #[error("a source's series_count must be greater than zero")] + ZeroSeriesCount, +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + use crate::accuracy::AccuracyTarget; + use crate::stages::ValueSign; + use crate::workload::*; + + #[test] + fn canonical_workloads_have_a_strict_json_round_trip() { + let workload = PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: None, + repeating_queries: Some(vec![RepeatingEntry { + query: Query("rate(requests_total[5m])".into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), + requirements: QueryRequirements { + accuracy: AccuracyTarget::max_error(0.01).failure_probability(0.001), + max_latency_ms: Some(100.0), + }, + predictability: Predictability::Predictable { known_at: None }, + range_evaluation: None, + }]), + }, + data_workload: Some(DataWorkload { + arrival: DataArrival::Ingesting { + rows_per_second: 1_000.0, + max_out_of_order_ms: None, + }, + sources: BTreeMap::from([( + "requests_total".to_string(), + SourceFacts { + series_count: Some(10_000), + metric_type: Some(PrometheusMetricType::Counter), + column_signs: BTreeMap::from([( + "value".to_string(), + ValueSign::NonNegative, + )]), + sample_interval: Some(DurationMs(15_000)), + stream_norms: Some(StreamNorms { + l1: Some(1e9), + l2: None, + }), + max_distinct_per_evaluation: Some(100), + distinct_values_per_label: BTreeMap::from([ + ("job".to_string(), 10), + ("instance".to_string(), 1_000), + ]), + distinct_values_per_label_set: vec![LabelSetDistinctValues { + labels: vec!["job".to_string(), "instance".to_string()], + distinct_values: 1_000, + }], + rows_held: Some(1_000_000), + }, + )]), + }), + }; + + let json = serde_json::to_value(&workload).expect("serialize workload"); + assert_eq!( + json["data_workload"], + serde_json::json!({ + "arrival": {"ingesting": {"rows_per_second": 1000.0}}, + "sources": {"requests_total": { + "series_count": 10000, + "metric_type": "counter", + "column_signs": {"value": "non_negative"}, + "sample_interval": 15000, + "stream_norms": {"l1": 1e9, "l2": null}, + "max_distinct_per_evaluation": 100, + "distinct_values_per_label": {"instance": 1000, "job": 10}, + "distinct_values_per_label_set": [ + {"labels": ["job", "instance"], "distinct_values": 1000} + ], + "rows_held": 1000000 + }} + }) + ); + let decoded: PlanningWorkload = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(decoded, workload); + + let text = serde_json::to_string(&workload).unwrap(); + let with_unknown = text.replacen("{", "{\"unknown\":true,", 1); + assert!(serde_json::from_str::(&with_unknown).is_err()); + + // Omitted facts default to unknown. + let minimal: DataWorkload = serde_json::from_value(serde_json::json!({ + "arrival": {"mixed": {"rows_per_second": 5.0}}, + "sources": {"flows": {"series_count": 7}} + })) + .unwrap(); + assert_eq!( + minimal.sources["flows"], + SourceFacts { + series_count: Some(7), + ..SourceFacts::default() + } + ); + } +} diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs new file mode 100644 index 000000000..e5d6f529b --- /dev/null +++ b/crates/types/src/workload/query.rs @@ -0,0 +1,822 @@ +//! The query workload: each query, its requirements and its recurrence. + +use crate::accuracy::AccuracyTarget; +use serde::{Deserialize, Serialize}; + +use super::data::validate_rate; +use super::{Rate, WorkloadError}; + +/// A raw query string in its source language, before any parsing. +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Query(pub String); + +/// How often a repeating query fires, in milliseconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct RepetitionInterval(pub u32); + +/// Milliseconds since the Unix epoch. Workload timestamps use one explicit +/// representation so schedules, observations, and time selections agree. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct TimestampMs(pub u64); + +/// A non-negative duration in milliseconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct DurationMs(pub u64); + +/// SQL dialect variant — different dialects have different syntax and +/// function sets that affect how the query string is parsed. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SqlDialect { + #[serde(rename = "datafusion_sql")] + DataFusionSQL, + #[serde(rename = "clickhouse_sql")] + ClickhouseSQL, + #[serde(rename = "elastic_sql")] + ElasticSQL, +} + +/// Source language of every query in the workload. +/// All queries in a single `QueryWorkload` share the same language. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum QueryLanguage { + #[serde(rename = "promql")] + PromQL, + #[serde(rename = "metricsql")] + MetricsQL, + #[serde(rename = "sql")] + SQL(SqlDialect), + #[serde(rename = "datafusion")] + DataFusion, + #[serde(rename = "elastic_dsl")] + ElasticDSL, +} + +/// Independent accuracy and response-latency constraints attached to one +/// query in the workload. +/// +/// JSON: `{"accuracy": {"max_error": 0.01, "accuracy_metric": "rank"}, "max_latency_ms": +/// 200.0}`. An omitted `accuracy` is [`AccuracyTarget::exact`] and an +/// omitted `max_latency_ms` imposes no bound. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct QueryRequirements { + /// The accuracy the query's answer must meet. + #[serde(default = "AccuracyTarget::exact")] + pub accuracy: AccuracyTarget, + /// The longest one evaluation may take, in milliseconds; `None` for no + /// bound. + #[serde(default)] + pub max_latency_ms: Option, +} + +impl Default for QueryRequirements { + fn default() -> Self { + Self { + accuracy: AccuracyTarget::exact(), + max_latency_ms: None, + } + } +} + +/// How far in advance the planner knows that a query will be executed. +/// This describes knowledge of the query, not how often it runs. +/// +/// # Example +/// +/// A report announced at `1_000` ms and executed later is predictable; +/// an interactive query typed by a user is ad hoc. +/// +/// ``` +/// use asap_types::workload::{Predictability, TimestampMs}; +/// +/// let report = Predictability::Predictable { +/// known_at: Some(TimestampMs(1_000)), +/// }; +/// let exploration = Predictability::AdHoc; +/// ``` +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub enum Predictability { + AdHoc, + Predictable { + known_at: Option, + }, + #[default] + Unknown, +} + +/// Evidence-backed forecast used when recurrence is not a fixed interval or +/// an explicit schedule. Demand is normalized to invocations per second at +/// the input boundary, so this type never mixes counts and rates. +/// +/// # Example +/// +/// This estimate predicts ten executions per second. It is usable for five +/// minutes after `observed_at`. +/// +/// ``` +/// use asap_types::workload::{DemandEstimate, DurationMs, Rate, TimestampMs}; +/// +/// let estimate = DemandEstimate { +/// expected_rate: Rate(10.0), +/// observed_at: Some(TimestampMs(60_000)), +/// valid_for: Some(DurationMs(5 * 60 * 1_000)), +/// }; +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DemandEstimate { + pub expected_rate: Rate, + pub observed_at: Option, + pub valid_for: Option, +} + +/// How a repeated query is expected to recur. +/// +/// # Example +/// +/// ``` +/// use asap_types::workload::{RepeatedDemand, RepetitionInterval, TimestampMs}; +/// +/// let dashboard = RepeatedDemand::FixedInterval(RepetitionInterval(10_000)); +/// let scheduled = RepeatedDemand::Scheduled(vec![ +/// TimestampMs(100_000), +/// TimestampMs(200_000), +/// ]); +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RepeatedDemand { + FixedInterval(RepetitionInterval), + /// Fixed cadence with a known evaluation phase. The timestamp is any + /// evaluation endpoint on that cadence, not necessarily its first run. + FixedIntervalAt { + interval: RepetitionInterval, + evaluation_phase: TimestampMs, + }, + Scheduled(Vec), + EstimatedRate(DemandEstimate), +} + +/// Normalized execution recurrence shared by batch and repeating workload +/// entries. This is independent of [`Predictability`]: a one-time query may +/// be known in advance or ad hoc, and a repeated query may still be uncertain. +/// +/// # Example +/// +/// ``` +/// use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval, TimestampMs}; +/// +/// let scheduled_once = QueryRecurrence::OneTime { +/// invocations: 1, +/// execute_at: Some(TimestampMs(100_000)), +/// }; +/// let every_minute = QueryRecurrence::Repeated( +/// RepeatedDemand::FixedInterval(RepetitionInterval(60_000)), +/// ); +/// ``` +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub enum QueryRecurrence { + OneTime { + invocations: u64, + execute_at: Option, + }, + Repeated(RepeatedDemand), + #[default] + Unknown, +} + +impl QueryRecurrence { + /// The fixed interval at which the query repeats, in milliseconds: + /// `Some` only for a fixed-interval repetition with a positive interval. + pub fn fixed_interval_ms(&self) -> Option { + match self { + QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(interval) + | RepeatedDemand::FixedIntervalAt { interval, .. }, + ) => Some(u64::from(interval.0)).filter(|&ms| ms > 0), + _ => None, + } + } +} + +/// The most evaluation steps one range request may have, as Prometheus +/// allows for `/api/v1/query_range` (it rejects a request with more). +pub(crate) const MAX_RANGE_EVALUATION_STEPS: u64 = 11_000; + +/// How one request of a query is answered as a range request, as +/// Prometheus' `/api/v1/query_range` does: it evaluates the query at +/// `start = end − span_ms` and every `step_ms` after, up to and including +/// the request's `end`, e.g. a dashboard panel over the last hour at +/// 1-minute steps (`...&start=now-1h&end=now&step=60`). `span_ms` must be a +/// whole number of steps, so the last step is the request's end. +/// [`QueryRecurrence`] still says how often a request runs. +/// +/// JSON: `{"step_ms": 60000, "span_ms": 3600000}`. +/// +/// # Example +/// +/// ``` +/// use asap_types::workload::RangeEvaluation; +/// +/// let last_hour_by_minute = RangeEvaluation { step_ms: 60_000, span_ms: 3_600_000 }; +/// assert_eq!(last_hour_by_minute.evaluations_per_request(), 61); +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RangeEvaluation { + /// Time between consecutive evaluation steps of one request, in + /// milliseconds; must be positive. + pub step_ms: u64, + /// How far before the request's end its first step is, in + /// milliseconds: a multiple of `step_ms`, 0 for one step per request. + pub span_ms: u64, +} + +impl RangeEvaluation { + /// Steps one request evaluates, `span_ms / step_ms + 1`: 61 for a 1 h + /// span at 1 min steps. Stage 3 prices each step as one evaluation. + pub fn evaluations_per_request(&self) -> u64 { + self.span_ms / self.step_ms.max(1) + 1 + } +} + +/// The greatest common divisor of `a` and `b` (`a` when `b` is 0): the +/// grid two cadences in milliseconds share. +pub fn greatest_common_divisor(a: u64, b: u64) -> u64 { + if b == 0 { + a + } else { + greatest_common_divisor(b, a % b) + } +} + +/// One query submitted as part of a finite batch, including how many times it +/// will run and, when known, its execution time. +/// +/// # Example +/// +/// ``` +/// use asap_types::workload::*; +/// +/// let report = BatchEntry { +/// query: Query("SELECT count(*) FROM events".into()), +/// requirements: QueryRequirements::default(), +/// predictability: Predictability::Predictable { +/// known_at: Some(TimestampMs(1_000)), +/// }, +/// invocations: 1, +/// execute_at: Some(TimestampMs(10_000)), +/// range_evaluation: None, +/// }; +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BatchEntry { + pub query: Query, + pub requirements: QueryRequirements, + pub predictability: Predictability, + pub invocations: u64, + pub execute_at: Option, + /// How each invocation is answered as a range request: `None` (the + /// default) for one evaluation per invocation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub range_evaluation: Option, +} + +/// One query with repeated demand. Its recurrence does not imply that the +/// queried data is continuously ingesting; that is described separately by +/// [`DataArrival`](crate::workload::DataArrival). +/// +/// # Example +/// +/// ``` +/// use asap_types::workload::*; +/// +/// let dashboard = RepeatingEntry { +/// query: Query("rate(requests[5m])".into()), +/// demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), +/// requirements: QueryRequirements::default(), +/// predictability: Predictability::Predictable { known_at: None }, +/// // Each refresh evaluates the last hour at 1-minute steps. +/// range_evaluation: Some(RangeEvaluation { step_ms: 60_000, span_ms: 3_600_000 }), +/// }; +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RepeatingEntry { + pub query: Query, + pub demand: RepeatedDemand, + pub requirements: QueryRequirements, + pub predictability: Predictability, + /// How each request is answered as a range request: `None` (the + /// default) for one evaluation per request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub range_evaluation: Option, +} + +/// Planner-facing normalized form of either [`BatchEntry`] or +/// [`RepeatingEntry`]. It keeps requirements, predictability and recurrence +/// as separate axes; the event time a query reads is in its text. Planning takes one per query, in +/// root order: Stage 2 (physical ASAP-aware optimization) reads the +/// recurrence and predictability, Stage 3 (plan selection) the accuracy +/// target, the latency bound and the recurrence. +/// +/// # Example +/// +/// ``` +/// use asap_types::workload::*; +/// +/// let entry = QueryWorkloadEntry { +/// query: Query("rate(requests[5m])".into()), +/// requirements: QueryRequirements::default(), +/// predictability: Predictability::Predictable { known_at: None }, +/// recurrence: QueryRecurrence::Repeated( +/// RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), +/// ), +/// range_evaluation: None, +/// }; +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct QueryWorkloadEntry { + /// The query text in the workload's language. + pub query: Query, + /// Its accuracy target and latency bound (default: exact, no bound). + pub requirements: QueryRequirements, + /// How far ahead it is known: a build may run at ingestion time only for + /// `Predictable` queries (default: unknown). + pub predictability: Predictability, + /// How often it runs: sets how often each node is priced, and only + /// repeated queries may keep state from ingestion time (default: unknown). + pub recurrence: QueryRecurrence, + /// How each run is answered as a range request, one evaluation per + /// step; `None` (the default) for one evaluation per run. Stage 1 and + /// Stage 2 keep tumbling-window widths on its step grid, and Stage 3 prices + /// [`RangeEvaluation::evaluations_per_request`] evaluations per run. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub range_evaluation: Option, +} + +impl QueryWorkloadEntry { + /// The grid every evaluation of this query falls on, in milliseconds, + /// anchored at the epoch (every evaluation time is a multiple of it): + /// its fixed interval ([`QueryRecurrence::fixed_interval_ms`]), or with + /// a range evaluation of more than one step the greatest common divisor + /// of the interval and the step, since each run evaluates at every step + /// up to its end; with a declared evaluation phase + /// ([`RepeatedDemand::FixedIntervalAt`]) also the greatest common + /// divisor with that phase (every 20 s at phase 5 s is on a 5 s grid). + /// `None` when it does not repeat at a positive fixed interval. + pub fn evaluation_grid_ms(&self) -> Option { + let interval_ms = self.recurrence.fixed_interval_ms()?; + let step_ms = match self.range_evaluation { + Some(range) if range.step_ms > 0 && range.evaluations_per_request() > 1 => { + greatest_common_divisor(interval_ms, range.step_ms) + } + _ => interval_ms, + }; + let phase_ms = match &self.recurrence { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + evaluation_phase, .. + }) => evaluation_phase.0, + _ => 0, + }; + Some(greatest_common_divisor(step_ms, phase_ms)) + } +} + +impl From<&BatchEntry> for QueryWorkloadEntry { + fn from(entry: &BatchEntry) -> Self { + Self { + query: entry.query.clone(), + requirements: entry.requirements.clone(), + predictability: entry.predictability.clone(), + recurrence: QueryRecurrence::OneTime { + invocations: entry.invocations, + execute_at: entry.execute_at, + }, + range_evaluation: entry.range_evaluation, + } + } +} + +impl From<&RepeatingEntry> for QueryWorkloadEntry { + fn from(entry: &RepeatingEntry) -> Self { + Self { + query: entry.query.clone(), + requirements: entry.requirements.clone(), + predictability: entry.predictability.clone(), + recurrence: QueryRecurrence::Repeated(entry.demand.clone()), + range_evaluation: entry.range_evaluation, + } + } +} + +impl DemandEstimate { + /// Whether this estimate was already observed and has not expired at + /// `now_ms`. A validity duration without an observation time is not a + /// usable freshness contract. + #[cfg(test)] + fn is_fresh_at(&self, now_ms: u64) -> bool { + match (self.observed_at, self.valid_for) { + (Some(observed), _) if observed.0 > now_ms => false, + (Some(observed), Some(valid_for)) => now_ms <= observed.0.saturating_add(valid_for.0), + (None, Some(_)) => false, + _ => true, + } + } +} + +/// `query_batch` and `repeating_queries` may both be present. [`Self::entries`] +/// normalizes them into one ordered stream without conflating recurrence with +/// data arrival. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct QueryWorkload { + /// Source language shared by all queries in this workload. + pub language: QueryLanguage, + /// One-shot queries executed together as a batch. + pub query_batch: Option>, + /// Queries that repeat on a fixed interval. + pub repeating_queries: Option>, +} + +impl QueryWorkload { + /// One normalized entry stream, independent of the source's legacy + /// batch/repeating containers. Mixed workloads preserve both kinds. + pub fn entries(&self) -> impl Iterator + '_ { + self.query_batch + .iter() + .flatten() + .map(QueryWorkloadEntry::from) + .chain( + self.repeating_queries + .iter() + .flatten() + .map(QueryWorkloadEntry::from), + ) + } + + pub(crate) fn validate(&self) -> Result<(), WorkloadError> { + for entry in self.entries() { + validate_entry(&entry)?; + } + Ok(()) + } +} + +fn validate_entry(entry: &QueryWorkloadEntry) -> Result<(), WorkloadError> { + if let Some(ms) = entry.requirements.max_latency_ms { + if !ms.is_finite() || ms < 0.0 { + return Err(WorkloadError::InvalidLatency(ms)); + } + } + match &entry.recurrence { + QueryRecurrence::OneTime { invocations: 0, .. } => { + return Err(WorkloadError::ZeroInvocations) + } + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(0))) + | QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(0), + .. + }) => return Err(WorkloadError::ZeroRepetitionInterval), + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(schedule)) if schedule.is_empty() => { + return Err(WorkloadError::EmptySchedule) + } + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { + validate_rate(estimate.expected_rate)?; + } + _ => {} + } + if let Some(range) = entry.range_evaluation { + if range.step_ms == 0 { + return Err(WorkloadError::ZeroRangeEvaluationStep); + } + if range.span_ms % range.step_ms != 0 { + return Err(WorkloadError::RangeEvaluationSpanNotMultipleOfStep { + span_ms: range.span_ms, + step_ms: range.step_ms, + }); + } + let steps = range.evaluations_per_request(); + if steps > MAX_RANGE_EVALUATION_STEPS { + return Err(WorkloadError::TooManyRangeEvaluationSteps { steps }); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workload::*; + + /// Only a fixed-interval repetition with a positive interval has a + /// fixed interval; a zero interval, a schedule, an estimated rate, a + /// one-time query and an unknown recurrence have none. + #[test] + fn fixed_interval_ms_is_set_only_for_a_positive_fixed_interval() { + let repeated = |demand| QueryRecurrence::Repeated(demand); + assert_eq!( + repeated(RepeatedDemand::FixedInterval(RepetitionInterval(10_000))).fixed_interval_ms(), + Some(10_000) + ); + assert_eq!( + repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(60_000), + evaluation_phase: TimestampMs(5_000), + }) + .fixed_interval_ms(), + Some(60_000) + ); + assert_eq!( + repeated(RepeatedDemand::FixedInterval(RepetitionInterval(0))).fixed_interval_ms(), + None + ); + assert_eq!( + repeated(RepeatedDemand::Scheduled(vec![TimestampMs(1_000)])).fixed_interval_ms(), + None + ); + assert_eq!( + QueryRecurrence::OneTime { + invocations: 1, + execute_at: None, + } + .fixed_interval_ms(), + None + ); + assert_eq!(QueryRecurrence::Unknown.fixed_interval_ms(), None); + } + + /// Query requirements: an accuracy target (omitted: exact) and an + /// optional latency bound in ms (omitted: none). The old tagged forms + /// and unknown fields are rejected. + #[test] + fn query_requirements_json_is_a_target_and_an_optional_bound() { + let parse = |json: &str| serde_json::from_str::(json); + let empty = serde_json::to_value(parse("{}").unwrap()).unwrap(); + assert_eq!( + empty, + serde_json::json!({"accuracy": {"max_error": 0.0, "failure_probability": 0.0, + "accuracy_scope": "whole_result"}, "max_latency_ms": null}) + ); + let bounded = serde_json::to_value( + parse(r#"{"accuracy": {"max_error": 0.01}, "max_latency_ms": 200.0}"#).unwrap(), + ) + .unwrap(); + assert_eq!(bounded["accuracy"]["max_error"], 0.01); + assert_eq!(bounded["max_latency_ms"], 200.0); + for old in [ + r#"{"accuracy": "implicit_exact", "response_latency": "unspecified"}"#, + r#"{"accuracy": {"explicit": {"max_error": 0.01}}}"#, + r#"{"response_latency": {"explicit_max_ms": 100.0}}"#, + r#"{"max_latency_ms": 100.0, "unknown": 1}"#, + ] { + assert!(parse(old).is_err(), "{old}"); + } + } + + /// A repeating query with a zero fixed interval is rejected. + #[test] + fn zero_repetition_interval_is_rejected() { + let mut workload = base_workload(); + workload.repeating_queries = Some(vec![RepeatingEntry { + query: Query("dashboard".into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(0)), + requirements: QueryRequirements::default(), + predictability: Predictability::Predictable { known_at: None }, + range_evaluation: None, + }]); + assert_eq!( + workload.validate(), + Err(WorkloadError::ZeroRepetitionInterval) + ); + } + + /// A range evaluation is optional JSON on a repeating entry, carried to + /// its normalized entry; its step must be positive and a request may + /// have at most `MAX_RANGE_EVALUATION_STEPS` steps. An entry without + /// one serializes as before, with no `range_evaluation` key. + #[test] + fn range_evaluation_is_optional_carried_and_validated() { + let dashboard = |range: serde_json::Value| { + let mut entry = serde_json::json!({ + "query": "rate(requests_total[5m])", + "demand": {"fixed_interval": 30000}, + "requirements": {}, + "predictability": {"predictable": {"known_at": null}}, + }); + if !range.is_null() { + entry["range_evaluation"] = range; + } + serde_json::from_value::(entry).unwrap() + }; + let plain = dashboard(serde_json::Value::Null); + assert_eq!(plain.range_evaluation, None); + assert!(serde_json::to_value(&plain) + .unwrap() + .get("range_evaluation") + .is_none()); + + let hour = dashboard(serde_json::json!({"step_ms": 60000, "span_ms": 3600000})); + let range = RangeEvaluation { + step_ms: 60_000, + span_ms: 3_600_000, + }; + assert_eq!(hour.range_evaluation, Some(range)); + assert_eq!(range.evaluations_per_request(), 61); + let entry = QueryWorkloadEntry::from(&hour); + assert_eq!(entry.range_evaluation, Some(range)); + // Refreshes every 30 s and steps every 60 s: evaluations on a 30 s grid. + assert_eq!(entry.evaluation_grid_ms(), Some(30_000)); + assert_eq!( + QueryWorkloadEntry::from(&plain).evaluation_grid_ms(), + Some(30_000) + ); + // At a declared phase, also on that phase from the epoch: every + // minute at phase 125 s is on a 5 s grid; at phase 120 s, on 60 s. + let at_phase = |phase_ms| QueryWorkloadEntry { + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(60_000), + evaluation_phase: TimestampMs(phase_ms), + }), + range_evaluation: None, + ..entry.clone() + }; + assert_eq!(at_phase(125_000).evaluation_grid_ms(), Some(5_000)); + assert_eq!(at_phase(120_000).evaluation_grid_ms(), Some(60_000)); + // A one-step range (span 0) is one evaluation per request: its + // step does not narrow the grid (refreshing every 60 s at a 45 s + // step would otherwise give a 15 s grid for nothing). + let one_step = QueryWorkloadEntry { + range_evaluation: Some(RangeEvaluation { + step_ms: 45_000, + span_ms: 0, + }), + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( + RepetitionInterval(60_000), + )), + ..entry.clone() + }; + assert_eq!(one_step.evaluation_grid_ms(), Some(60_000)); + + let validate = |range: RangeEvaluation| { + let mut workload = base_workload(); + workload.repeating_queries = Some(vec![RepeatingEntry { + range_evaluation: Some(range), + ..hour.clone() + }]); + workload.validate() + }; + assert_eq!(validate(range), Ok(())); + assert_eq!( + validate(RangeEvaluation { + step_ms: 0, + span_ms: 60_000 + }), + Err(WorkloadError::ZeroRangeEvaluationStep) + ); + assert_eq!( + validate(RangeEvaluation { + step_ms: 1_000, + span_ms: 11_000_000 + }), + Err(WorkloadError::TooManyRangeEvaluationSteps { steps: 11_001 }) + ); + assert_eq!( + validate(RangeEvaluation { + step_ms: 1_000, + span_ms: 10_999_000 + }), + Ok(()) + ); + // Prometheus places steps forward from the start, so a span that is + // not a whole number of steps would put the last step off the end's + // grid: refreshing every 60 s at 60 s steps over 90 s evaluates at + // E − 90 s and E − 30 s. It is rejected. + assert_eq!( + validate(RangeEvaluation { + step_ms: 60_000, + span_ms: 90_000 + }), + Err(WorkloadError::RangeEvaluationSpanNotMultipleOfStep { + span_ms: 90_000, + step_ms: 60_000 + }) + ); + let unknown = serde_json::json!({"step_ms": 1, "span_ms": 0, "start": 0}); + assert!(serde_json::from_value::(unknown).is_err()); + } + + fn base_workload() -> QueryWorkload { + QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: None, + repeating_queries: None, + } + } + + #[test] + fn entries_keep_each_entrys_recurrence() { + let mut workload = base_workload(); + workload.query_batch = Some(vec![BatchEntry { + query: Query("historical".into()), + requirements: QueryRequirements::default(), + predictability: Predictability::AdHoc, + invocations: 1, + execute_at: None, + range_evaluation: None, + }]); + workload.repeating_queries = Some(vec![RepeatingEntry { + query: Query("dashboard".into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), + requirements: QueryRequirements::default(), + predictability: Predictability::Predictable { known_at: None }, + range_evaluation: None, + }]); + + let entries: Vec<_> = workload.entries().collect(); + assert_eq!(entries.len(), 2); + assert!(matches!( + entries[0].recurrence, + QueryRecurrence::OneTime { .. } + )); + assert!(matches!( + entries[1].recurrence, + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(_)) + )); + workload.validate().unwrap(); + } + + #[test] + fn future_demand_estimates_are_not_fresh() { + let estimate = DemandEstimate { + expected_rate: Rate(1.0), + observed_at: Some(TimestampMs(2_000)), + valid_for: Some(DurationMs(1_000)), + }; + assert!(!estimate.is_fresh_at(1_999)); + assert!(estimate.is_fresh_at(2_000)); + } + + #[test] + fn query_language_wire_names_do_not_split_acronyms() { + let cases = [ + (QueryLanguage::PromQL, serde_json::json!("promql")), + (QueryLanguage::MetricsQL, serde_json::json!("metricsql")), + ( + QueryLanguage::SQL(SqlDialect::DataFusionSQL), + serde_json::json!({"sql": "datafusion_sql"}), + ), + ( + QueryLanguage::SQL(SqlDialect::ClickhouseSQL), + serde_json::json!({"sql": "clickhouse_sql"}), + ), + (QueryLanguage::ElasticDSL, serde_json::json!("elastic_dsl")), + ]; + for (language, expected) in cases { + assert_eq!(serde_json::to_value(language).unwrap(), expected); + } + } + + #[test] + fn struct_enum_variants_reject_unknown_fields() { + assert!( + serde_json::from_value::(serde_json::json!({ + "one_time": {"invocations": 1, "execute_at": null, "typo": true} + })) + .is_err() + ); + assert!(serde_json::from_value::(serde_json::json!({ + "predictable": {"known_at": null, "typo": true} + })) + .is_err()); + assert!(serde_json::from_value::(serde_json::json!({ + "max_error": 0.1, "failure_probability": 0.01, "typo": true + })) + .is_err()); + } + + #[test] + fn estimated_demand_validates_its_rate() { + let mut workload = base_workload(); + workload.repeating_queries = Some(vec![RepeatingEntry { + query: Query("estimated".into()), + demand: RepeatedDemand::EstimatedRate(DemandEstimate { + expected_rate: Rate(-1.0), + observed_at: None, + valid_for: None, + }), + requirements: QueryRequirements::default(), + predictability: Predictability::Unknown, + range_evaluation: None, + }]); + assert_eq!(workload.validate(), Err(WorkloadError::InvalidRate(-1.0))); + } +} diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index e95aeb9cf..bf56559c0 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,12 +2,10 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. -mod materialized_view; mod physical_export; mod planner_vocabulary; mod schema_rebuilding; mod structure_contract; -mod summary_coverage; mod summary_merge_structure; mod tumbling_window_chain; mod workload_compatibility; diff --git a/crates/types/tests/materialized_view.rs b/crates/types/tests/materialized_view.rs deleted file mode 100644 index 0576ca187..000000000 --- a/crates/types/tests/materialized_view.rs +++ /dev/null @@ -1,512 +0,0 @@ -//! A DAG's materialized views and their retention -//! (`asap_types::ir::materialized_view`): two one-minute KLL tumbling windows of `m` -//! kept at ingestion, merged into a view refreshed on first read. - -use std::collections::HashMap; -use std::rc::Rc; -use std::time::Duration; - -use asap_types::ir::materialized_view::{ - InstallableDagError, MaterializedViewRefresh, RefreshWindow, -}; -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; -use asap_types::ir::operator::TimeShift; -use asap_types::ir::physical_export::{ - compile_physical_asap_dag, compile_physical_asap_workload, PhysicalASAPDAG, -}; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchKind, - SketchParams, SummaryUpdate, -}; -use asap_types::ir::{ - apply_materialization_timings, apply_materialization_timings_to_roots, ASAPOp, - MaterializationAssignment, NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, QueryRoot, - TimeRangeKind, -}; -use asap_types::physical::{Materialization, Medium}; - -/// A KLL of `m`'s values over `(T - shift - 1 min, T - shift]`. -fn tumbling_window(scan: &Rc, shift_ms: i64) -> Rc { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift_ms, - at: None, - }, - child: Rc::clone(scan), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(60), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap(); - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: range, - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() -} - -/// `m` with columns `ts`, `value` and `job`. -fn scan_m() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -/// The tumbling windows at offsets 0 and 1 min kept at ingestion, their merge kept -/// across evaluations, exported: `(dag, merge's node id)`. -fn tumbling_windows_and_kept_merge() -> (PhysicalASAPDAG, usize) { - let scan = scan_m(); - let tumbling_windows = [tumbling_window(&scan, 0), tumbling_window(&scan, 60_000)]; - let merge = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: tumbling_windows.to_vec(), - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - for tumbling_window in &tumbling_windows { - assignment.set( - tumbling_window, - Materialization::MatAtIngestion(Medium::Memory), - ); - } - assignment.set( - &merge, - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ); - let timed = - apply_materialization_timings(&merge, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - let merge_id = dag.roots[0]; - (dag, merge_id) -} - -/// A view without `retention_ms` (a DAG before installation) has no -/// materialized views to list. -#[test] -fn a_view_without_retention_is_refused() { - let (dag, _) = tumbling_windows_and_kept_merge(); - assert!(matches!( - dag.materialized_views(), - Err(InstallableDagError::ViewWithoutRetention { .. }) - )); -} - -/// The merge, refreshed on first read for each step of a one-hour range, -/// is kept for its two minutes plus the hour; each tumbling window it reads is kept -/// that long too, since the merge may be refreshed for its oldest step. -/// The tumbling windows read `m` at ingestion, so no raw data is kept for queries. -#[test] -fn retention_propagates_from_a_view_refreshed_on_first_read_to_its_inputs() { - let (mut dag, merge_id) = tumbling_windows_and_kept_merge(); - let spans = HashMap::from([(merge_id, 3_600_000)]); - dag.set_retention(&HashMap::new(), &spans).unwrap(); - let views = dag.materialized_views().unwrap(); - let (merge, tumbling_windows) = views.split_last().unwrap(); - assert_eq!(merge.node_id, merge_id); - assert_eq!(merge.refresh(), MaterializedViewRefresh::OnFirstRead); - assert_eq!(merge.retention_ms(), 120_000 + 3_600_000); - assert_eq!(tumbling_windows.len(), 2); - for tumbling_window in tumbling_windows { - assert_eq!( - tumbling_window.refresh(), - MaterializedViewRefresh::AtIngestion - ); - assert_eq!(tumbling_window.retention_ms(), merge.retention_ms()); - let node = dag - .nodes - .iter() - .find(|n| n.id == tumbling_window.node_id) - .unwrap(); - assert_eq!(node.retention_ms, Some(tumbling_window.retention_ms())); - } - assert_eq!(dag.raw_data_retention().unwrap(), []); -} - -/// A view kept for less than its data reaches back (the older tumbling window, which -/// reaches 2 minutes back, kept for 1 minute) is refused. -#[test] -fn a_view_kept_shorter_than_its_reach_is_refused() { - let (mut dag, _) = tumbling_windows_and_kept_merge(); - let views = dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let older = views - .iter() - .find(|v| { - v.inputs - .raw_data_windows() - .first() - .map(|w| w.end_offset_ms()) - == Some(60_000) - }) - .unwrap() - .node_id; - dag.nodes - .iter_mut() - .find(|n| n.id == older) - .unwrap() - .retention_ms = Some(60_000); - assert_eq!( - dag.materialized_views(), - Err(InstallableDagError::ViewRetentionTooShort { node_id: older }) - ); -} - -/// A quantile sketch of `algorithm` (KLL k = 200 or DDSketch α = 0.01). -fn quantile_sketch(algorithm: SketchAlgorithm) -> FieldDataType { - let params = match algorithm { - SketchAlgorithm::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - _ => SketchParams::DDSketch { alpha: 0.01 }, - }; - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) -} - -/// `tumbling_window_count` tumbling windows of `tumbling_window_width_ms` of `family` per `job` over `input`. -fn chain( - input: &Rc, - family: FieldDataType, - tumbling_window_width_ms: u64, - tumbling_window_count: u32, -) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(input), - family, - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - tumbling_window_width_ms, - tumbling_window_count, - })) - .unwrap() -} - -/// Three queries over one `Scan` of `m`, each merging a chain kept at -/// ingestion: a KLL of six 10 s tumbling windows, a DDSketch of thirty 10 s tumbling windows, and -/// a KLL of five 60 s tumbling windows. The DAG Stage 3 costs (chains unexpanded) and -/// the installed DAG (chains expanded into tumbling windows, retention set). -fn three_chains_over_one_scan() -> (PhysicalASAPDAG, PhysicalASAPDAG) { - let scan = scan_m(); - let chains = [ - chain(&scan, quantile_sketch(SketchAlgorithm::Kll), 10_000, 6), - chain( - &scan, - quantile_sketch(SketchAlgorithm::DDSketch), - 10_000, - 30, - ), - chain(&scan, quantile_sketch(SketchAlgorithm::Kll), 60_000, 5), - ]; - let mut assignment = MaterializationAssignment::default(); - let roots: Vec = chains - .iter() - .map(|chain| { - assignment.set(chain, Materialization::MatAtIngestion(Medium::Memory)); - QueryRoot::Operator( - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(chain)], - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(), - ) - }) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - let unexpanded = compile_physical_asap_workload(&timed).unwrap(); - let expanded = expand_tumbling_window_chains_in_workload(&timed).unwrap(); - let mut installed = compile_physical_asap_workload(&expanded.roots).unwrap(); - installed - .set_retention(&HashMap::new(), &HashMap::new()) - .unwrap(); - (unexpanded, installed) -} - -/// The width and sketch of the tumbling windows `node` of `dag` builds. -fn tumbling_window_shape(dag: &PhysicalASAPDAG, node: usize) -> (u64, SketchAlgorithm) { - let payload = &dag.nodes.iter().find(|n| n.id == node).unwrap().payload; - let (family, width_ms) = match payload { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - .. - }) => (family, *tumbling_window_width_ms), - Operator::ASAP(ASAPOp::SummaryAgg { family, child, .. }) => { - let range = &dag.nodes.iter().find(|n| n.id == *child).unwrap().payload; - let Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) = range else { - panic!("a tumbling window reads a TimeRange") - }; - (family, range.as_millis() as u64) - } - other => panic!("not a tumbling window: {other:?}"), - }; - let FieldDataType::Sketch(kind, _) = family else { - panic!("a sketch") - }; - (width_ms, *kind.algorithm()) -} - -/// A chain kept at ingestion has its newest tumbling window's refresh window on the -/// DAG Stage 3 costs, and each of its expanded tumbling windows has the same one. -#[test] -fn a_chain_and_each_of_its_tumbling_windows_have_one_refresh_window() { - let (unexpanded, installed) = three_chains_over_one_scan(); - let window_of = |width_ms| RefreshWindow { - width_ms, - phase_ms: 0, - }; - let chains: Vec = unexpanded - .nodes - .iter() - .filter(|node| { - matches!( - node.payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .map(|node| node.id) - .collect(); - assert_eq!(chains.len(), 3); - for chain in chains { - let (width_ms, _) = tumbling_window_shape(&unexpanded, chain); - assert_eq!(unexpanded.refresh_window(chain), Some(window_of(width_ms))); - } - let views = installed.materialized_views().unwrap(); - assert_eq!(views.len(), 6 + 30 + 5); - for view in &views { - let (width_ms, _) = tumbling_window_shape(&installed, view.node_id); - assert_eq!( - installed.refresh_window(view.node_id), - Some(window_of(width_ms)) - ); - } -} - -/// The 10 s KLL and 10 s DDSketch tumbling windows are one refresh group, the 60 s -/// KLL tumbling windows another; the merges, at query time, are in none. -#[test] -fn tumbling_windows_of_one_width_form_one_refresh_group() { - let (_, installed) = three_chains_over_one_scan(); - let groups = installed.ingestion_refresh_groups().unwrap(); - let shapes: Vec<_> = groups - .iter() - .map(|group| { - let mut shapes: Vec<_> = group - .views - .iter() - .map(|view| tumbling_window_shape(&installed, view.node_id)) - .collect(); - shapes.sort_by_key(|(width_ms, algorithm)| (*width_ms, format!("{algorithm:?}"))); - shapes.dedup(); - (group.refresh_window, shapes) - }) - .collect(); - assert_eq!( - shapes, - [ - ( - Some(RefreshWindow { - width_ms: 10_000, - phase_ms: 0 - }), - vec![ - (10_000, SketchAlgorithm::DDSketch), - (10_000, SketchAlgorithm::Kll) - ] - ), - ( - Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 0 - }), - vec![(60_000, SketchAlgorithm::Kll)] - ), - ] - ); - assert_eq!(groups[0].views.len(), 6 + 30); - assert_eq!(groups[1].views.len(), 5); -} - -/// A tumbling window whose window ends 30 s after a multiple of its 60 s width (a -/// `TimeShift(90 s)` below its range) is refreshed on another grid than a -/// tumbling window at no offset: another refresh group. -#[test] -fn a_shifted_tumbling_window_has_its_own_phase() { - let scan = scan_m(); - let tumbling_windows = [tumbling_window(&scan, 0), tumbling_window(&scan, 90_000)]; - let mut assignment = MaterializationAssignment::default(); - for tumbling_window in &tumbling_windows { - assignment.set( - tumbling_window, - Materialization::MatAtIngestion(Medium::Memory), - ); - } - let roots: Vec = tumbling_windows - .iter() - .map(|tumbling_window| QueryRoot::Operator(Rc::clone(tumbling_window))) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); - dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let mut windows: Vec<(u64, u64)> = dag - .ingestion_refresh_groups() - .unwrap() - .iter() - .map(|group| { - let window = group.refresh_window.unwrap(); - (window.width_ms, window.phase_ms) - }) - .collect(); - windows.sort_unstable(); - assert_eq!(windows, [(60_000, 0), (60_000, 30_000)]); -} - -/// Two views kept at ingestion over rows with no time window (`Values`) -/// have no refresh window: each is a group of its own. -#[test] -fn views_of_unbounded_windows_are_groups_of_their_own() { - let rows = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![], - schema: scan_m().schema.clone(), - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - let roots: Vec = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] - .into_iter() - .map(|algorithm| { - let build = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(&rows), - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap(); - assignment.set(&build, Materialization::MatAtIngestion(Medium::Memory)); - QueryRoot::Operator(build) - }) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); - dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let groups = dag.ingestion_refresh_groups().unwrap(); - assert_eq!(groups.len(), 2); - for group in &groups { - assert_eq!(group.refresh_window, None); - assert_eq!(group.views.len(), 1); - } -} - -/// A rollup across jobs of a tumbling window, both kept at ingestion, reads the tumbling window's -/// window: one refresh group, the tumbling window listed before the rollup that reads -/// it, so a run whose roots hold both computes the tumbling window in the run. -#[test] -fn a_view_reading_a_view_of_its_group_follows_it() { - let scan = scan_m(); - let tumbling_window = tumbling_window(&scan, 0); - let rollup = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&tumbling_window)], - newest_tumbling_windows: None, - group_by: Some(vec![]), - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - for view in [&tumbling_window, &rollup] { - assignment.set(view, Materialization::MatAtIngestion(Medium::Memory)); - } - let timed = apply_materialization_timings_to_roots( - &[QueryRoot::Operator(Rc::clone(&rollup))], - &assignment, - ) - .unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); - dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let groups = dag.ingestion_refresh_groups().unwrap(); - assert_eq!(groups.len(), 1); - let [tumbling_window_view, rollup_view] = groups[0].views.as_slice() else { - panic!("{groups:#?}") - }; - assert_eq!(rollup_view.node_id, dag.roots[0]); - assert_eq!( - rollup_view.inputs.materialized_views, - [tumbling_window_view.node_id] - ); - assert_eq!( - groups[0].refresh_window, - Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 0 - }) - ); -} - -/// On the DAG Stage 3 costs (chains unexpanded), a node above a chain has -/// the window of all the chain's tumbling windows: a merge of six 10 s tumbling windows, refreshed -/// at ingestion, has refresh window (60 s, 0). -#[test] -fn a_node_above_an_unexpanded_chain_has_the_window_of_its_tumbling_windows() { - let (unexpanded, _) = three_chains_over_one_scan(); - let merge_of_kll_10s = unexpanded.roots[0]; - assert_eq!( - unexpanded.refresh_window(merge_of_kll_10s), - Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 0 - }) - ); -} - -/// A non-chain view (a KLL over `TimeRange(1 min)` shifted 15 s) has the -/// same refresh window, (60 s, 15 s), on a DAG of its nodes alone (no -/// edges, what Stage 3 builds from candidates): `view_windows` walks each -/// operator's inputs. -#[test] -fn a_view_has_its_refresh_window_on_a_dag_without_edges() { - let view = tumbling_window(&scan_m(), 15_000); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&view, Materialization::MatAtIngestion(Medium::Memory)); - let timed = apply_materialization_timings(&view, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let mut dag = compile_physical_asap_dag(&timed).unwrap(); - let view_id = dag.roots[0]; - let window = Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 15_000, - }); - assert_eq!(dag.refresh_window(view_id), window); - dag.edges.clear(); - assert_eq!(dag.refresh_window(view_id), window); -} diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index e2f701ffb..828ee1993 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -1,18 +1,13 @@ -//! A timed plan exports as a PhysicalASAPDAG that keeps timing. +//! A plan with its nodes' physical properties exports as a PhysicalASAPDAG +//! that keeps them. use asap_types::ir::operator::{Reduction, Source}; -use asap_types::ir::physical_export::{ - compile_physical_asap_dag, compile_physical_asap_workload, PhysicalASAPDAG, - PhysicalASAPDAGValidationError, -}; -use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; +use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{ExactKind, ExactParams, SummaryUpdate}; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, Operator, OperatorNode, QueryRoot, -}; -use asap_types::physical::{Materialization, Medium}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; +use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPDAGValidationError}; +use asap_types::stages::{Lifetime, PhysicalPropertiesError, StorageMedium}; +use asap_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; use std::rc::Rc; /// Scan(t) → SummaryAgg(sum by key) → FinalizeExactAccumulator, untimed. @@ -30,10 +25,9 @@ fn plan() -> Rc { .unwrap(); let state = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: scan, - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), input: SummaryUpdate::column(ColumnRef::Named("value".into())), reduction: Reduction::by(vec![0]), - grouping: Default::default(), filter: None, })) .unwrap(); @@ -43,107 +37,22 @@ fn plan() -> Rc { .unwrap() } -const ON_DISK: Materialization = Materialization::MatAtIngestion(Medium::Disk); - -/// `plan()` timed with its summary `MatAtIngestion` on disk. -fn timed_at_ingestion() -> Rc { - let root = plan(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(root.children()[0], ON_DISK); - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()).unwrap() -} - -/// A summary materialized at ingestion time is read at query time, and the -/// exported DAG says where to store it. -#[test] -fn timed_plan_exports_with_timing() { - let timed = timed_at_ingestion(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - dag.validate().unwrap(); - - let timings: Vec<_> = dag.nodes.iter().map(|n| n.output_state.timing).collect(); - assert_eq!( - timings, - vec![ - ExecutionTiming::IngestionTime, - ExecutionTiming::IngestionTime, - ExecutionTiming::QueryTime - ] - ); - let materializations: Vec<_> = dag.nodes.iter().map(|n| n.materialization).collect(); - assert_eq!( - materializations, - vec![ - Materialization::NotMaterialized, - ON_DISK, - Materialization::NotMaterialized - ] - ); - // Timing keeps what the summary covers. - assert!(timed.children()[0].coverage().is_some()); - - let decoded: PhysicalASAPDAG = - serde_json::from_str(&serde_json::to_string(&dag).unwrap()).unwrap(); - assert_eq!(decoded, dag); -} - -/// A query-time producer cannot feed an ingestion-time consumer. -#[test] -fn query_time_input_to_ingestion_is_rejected() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); - dag.nodes[0].output_state.timing = ExecutionTiming::QueryTime; - *dag.edges[0].data_state_mut() = dag.nodes[0].output_state; - assert!(matches!( - dag.validate(), - Err(PhysicalASAPDAGValidationError::QueryDependencyInIngestion { .. }) - )); -} - -/// A summary's timing must be the one its materialization implies: the -/// deployment cannot run a `MatAtIngestion` state at query time. -#[test] -fn materialization_inconsistent_with_timing_is_rejected() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); - dag.nodes[1].output_state.timing = ExecutionTiming::QueryTime; - *dag.edges[1].data_state_mut() = dag.nodes[1].output_state; - assert_eq!( - dag.validate(), - Err( - PhysicalASAPDAGValidationError::MaterializationTimingMismatch { - node: 1, - timing: ExecutionTiming::QueryTime, - materialization: ON_DISK, - } - ) - ); - dag.nodes[1].materialization = Materialization::MatAtQuery(Medium::Disk); - dag.validate().unwrap(); -} - -/// Untimed plans cannot be exported as physical plans. -#[test] -fn untimed_plan_is_rejected() { - assert!(compile_physical_asap_dag(&plan()).is_err()); -} - /// Two queries reading one summary state export once, with one root per query. #[test] fn batch_shares_the_summary_and_keeps_one_root_per_query() { - use asap_types::ir::physical_export::compile_physical_asap_workload; let first = plan(); let state = first.children()[0].clone(); let second = OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state, })) .unwrap(); - let assignment = MaterializationAssignment::default(); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = [first, second] - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache).unwrap()) + let roots: Vec = [first, second] + .into_iter() + .map(QueryRoot::Operator) .collect(); - let roots: Vec = timed.into_iter().map(QueryRoot::Operator).collect(); - let dag = compile_physical_asap_workload(&roots).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots) + .map(|(dag, _)| dag) + .unwrap(); dag.validate().unwrap(); assert_eq!(dag.roots.len(), 2); assert_eq!(dag.nodes.len(), 4, "scan and summary are exported once"); @@ -155,22 +64,18 @@ fn batch_shares_the_summary_and_keeps_one_root_per_query() { fn scalar_roots_export_in_batch_order_and_reference_nodes() { use asap_types::ir::scalar::ArithmeticOpKind; use asap_types::ir::{ExprSemantics, ScalarExpr}; - let timed = apply_materialization_timings( - &plan(), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); + let read = plan(); let scalar = ScalarExpr::Arithmetic { op: ArithmeticOpKind::Add, - left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&timed))), + left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&read))), right: Box::new(ScalarExpr::literal_f64(1.0)), semantics: ExprSemantics::Promql, }; - let dag = compile_physical_asap_workload(&[ + let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&[ QueryRoot::Scalar(scalar), - QueryRoot::Operator(Rc::clone(&timed)), + QueryRoot::Operator(Rc::clone(&read)), ]) + .map(|(dag, _)| dag) .unwrap(); assert_eq!(dag.nodes.len(), 3, "the read plan is exported once"); assert_eq!(dag.roots.len(), 1); @@ -187,29 +92,27 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { scalar_only.roots.clear(); scalar_only.validate().unwrap(); let valid_scalar_only = scalar_only.clone(); - // Materialization is checked on the nodes a scalar reads, too. + // Physical properties are checked on the nodes a scalar reads, too. let state = scalar_only .nodes .iter() .position(|node| { matches!( node.payload, - asap_types::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( + asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload::ASAP( ASAPOp::SummaryAgg { .. } ) ) }) .unwrap(); - scalar_only.nodes[state].materialization = ON_DISK; + scalar_only.nodes[state].output_state.timing = ExecutionTiming::IngestionTime; + scalar_only.nodes[state].lifetime = Lifetime::OneEvaluation(StorageMedium::Disk); assert_eq!( scalar_only.validate(), - Err( - PhysicalASAPDAGValidationError::MaterializationTimingMismatch { - node: scalar_only.nodes[state].id, - timing: ExecutionTiming::QueryTime, - materialization: ON_DISK, - } - ) + Err(PhysicalASAPDAGValidationError::PhysicalProperties { + node: scalar_only.nodes[state].id, + reason: PhysicalPropertiesError::OneEvaluationAtIngestion, + }) ); let mut missing = dag.clone(); @@ -247,9 +150,9 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { /// `ScalarRef` edge to it, as cost lowering reads it once. #[test] fn a_reference_read_twice_is_one_scalar_ref_edge() { - use asap_types::ir::physical_export::EdgeRole; use asap_types::ir::scalar::ArithmeticOpKind; use asap_types::ir::{ExprSemantics, ProjectItem, ScalarExpr}; + use asap_types::plan::physical_asap_dag::EdgeRole; let series = |metric: &str| { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { @@ -276,13 +179,7 @@ fn a_reference_read_twice_is_one_scalar_ref_edge() { child: series("a"), })) .unwrap(); - let timed = apply_materialization_timings( - &project, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(&project).unwrap(); dag.validate().unwrap(); let roles: Vec<_> = dag .edges @@ -298,9 +195,9 @@ fn a_reference_read_twice_is_one_scalar_ref_edge() { #[test] fn swapped_edge_roles_are_rejected() { use asap_types::ir::operator::JoinKind; - use asap_types::ir::physical_export::EdgeRole; use asap_types::ir::scalar::ScalarValue; use asap_types::ir::{Predicate, ScalarExpr}; + use asap_types::plan::physical_asap_dag::EdgeRole; let scan = |table: &str| { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { @@ -318,13 +215,7 @@ fn swapped_edge_roles_are_rejected() { right: scan("b"), })) .unwrap(); - let timed = apply_materialization_timings( - &join, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let mut dag = compile_physical_asap_dag(&timed).unwrap(); + let mut dag = asap_types::test_support::physical_asap_dag_at_query_time(&join).unwrap(); dag.validate().unwrap(); for edge in &mut dag.edges { edge.role = match edge.role { @@ -345,145 +236,16 @@ fn swapped_edge_roles_are_rejected() { #[test] fn an_empty_batch_is_an_error() { assert!(matches!( - compile_physical_asap_workload(&[]), - Err(ExecutionDataStateError::InvalidPhysicalDAG( - PhysicalASAPDAGValidationError::NoRoots - )) + asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&[]) + .map(|(dag, _)| dag), + Err(PhysicalASAPDAGValidationError::NoRoots) )); } -/// A summary over another summary exports a typed maintenance edge between -/// them, and the document round-trips and rejects unknown fields and versions. -#[test] -fn exports_summary_over_summary_as_typed_precompute_edges() { - use asap_types::ir::physical_export::{GroupingEdgeCompatibility, WindowEdgeCompatibility}; - use asap_types::ir::properties::ExecutionDataState; - use asap_types::test_support::compile_physical_asap_dag_with_node_ids; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let sum = |child, column| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate::column(column), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - })) - .unwrap() - }; - let inner = sum(scan, ColumnRef::SampleValue); - let outer = sum(Rc::clone(&inner), ColumnRef::Named("state".into())); - let root = OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: outer, - })) - .unwrap(); - let mut cache = NodeToTimedNodeCache::new(); - let at_ingestion = - MaterializationAssignment::with_default(Materialization::MatAtIngestion(Medium::Memory)); - let timed = apply_materialization_timings(&root, &at_ingestion, &mut cache).unwrap(); - let compiled = compile_physical_asap_dag_with_node_ids(&timed).unwrap(); - assert_eq!(compiled.node_ids.node_id(&timed), Some(3)); - assert_eq!( - compiled - .node_ids - .node_id(asap_types::test_support::timed(&cache, &inner).unwrap()), - Some(1) - ); - let dag = compiled.dag; - assert_eq!(dag.roots, vec![3]); - assert_eq!( - dag.nodes[1].output_state, - ExecutionDataState::INGESTION_SUMMARY - ); - assert_eq!( - dag.nodes[2].output_state, - ExecutionDataState::INGESTION_SUMMARY - ); - let dependency = dag - .edges - .iter() - .find(|e| e.producer == 1 && e.consumer == 2) - .unwrap(); - assert_eq!( - dependency.data_state(), - ExecutionDataState::INGESTION_SUMMARY - ); - assert_eq!(dependency.grouping(), GroupingEdgeCompatibility::Identical); - assert_eq!( - dependency.window(), - WindowEdgeCompatibility::RequiresAlignedTumblingWindowPhaseOrExactWindowEdgeResidual - ); - assert!(matches!( - dependency.intermediate_schema().fields[0].dtype, - FieldDataType::ExactAggregate(ExactKind::Sum, _) - )); - assert!(matches!( - dag.nodes[2].payload, - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - reduction: Reduction::Reduce(_), - .. - }) - )); - - let decoded: PhysicalASAPDAG = - serde_json::from_str(&serde_json::to_string(&dag).unwrap()).unwrap(); - assert_eq!(decoded, dag); - decoded.validate().unwrap(); - let mut unknown = serde_json::to_value(&dag).unwrap(); - unknown["unexpected"] = serde_json::json!(true); - assert!(serde_json::from_value::(unknown).is_err()); -} - -/// `retention_ms` is accepted on a materialized view (a node kept across -/// evaluations), survives a round trip, and is rejected on a node that is -/// neither a view nor a query-time scan (here an ingestion-time node). -#[test] -fn retention_is_only_on_materialized_views_and_query_time_scans() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); - let view = dag - .nodes - .iter() - .position(|node| asap_types::test_support::is_materialized_view(node.materialization)) - .expect("a node built at ingestion"); - let other = dag - .nodes - .iter() - .position(|node| { - !asap_types::test_support::is_materialized_view(node.materialization) - && node.output_state.timing == ExecutionTiming::IngestionTime - }) - .expect("an ingestion-time node that is not a view"); - dag.nodes[view].retention_ms = Some(60_000); - dag.validate().unwrap(); - let decoded: PhysicalASAPDAG = - serde_json::from_str(&serde_json::to_string(&dag).unwrap()).unwrap(); - assert_eq!(decoded, dag); - dag.nodes[other].retention_ms = Some(60_000); - assert_eq!( - dag.validate(), - Err(PhysicalASAPDAGValidationError::UnexpectedRetention( - dag.nodes[other].id - )) - ); -} - /// Node-id-keyed bindings serialize in numeric id order. #[test] fn physical_node_ids_serialize_in_deterministic_binding_order() { - use asap_types::ir::physical_export::PhysicalASAPNodeId; + use asap_types::plan::physical_asap_dag::PhysicalASAPNodeId; let mut bindings = std::collections::BTreeMap::::new(); bindings.insert(10, "materialization-10"); bindings.insert(2, "query-2"); @@ -493,46 +255,11 @@ fn physical_node_ids_serialize_in_deterministic_binding_order() { ); } -/// The plan with every node at query time is the same operators with -/// nothing materialized: what the default materialization exports. The -/// shared summary stays one node, and the scalar root that reads it keeps -/// its batch position. -#[test] -fn every_node_at_query_time_keeps_the_operators_and_stores_nothing() { - use asap_types::ir::scalar::ArithmeticOpKind; - use asap_types::ir::{ExprSemantics, ScalarExpr}; - let workload = |root: Rc| { - let scalar = ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Add, - left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&root))), - right: Box::new(ScalarExpr::literal_f64(1.0)), - semantics: ExprSemantics::Promql, - }; - compile_physical_asap_workload(&[ - QueryRoot::Operator(Rc::clone(&root)), - QueryRoot::Scalar(scalar), - QueryRoot::Operator(root), - ]) - .unwrap() - }; - let materialized = workload(timed_at_ingestion()); - let at_query_time = apply_materialization_timings( - &plan(), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!( - materialized.with_every_node_at_query_time().unwrap(), - workload(at_query_time) - ); -} - /// A window 600,000 ms wide ending 300,000 ms before T = 3,600,000 covers /// `(2,700,000, 3,300,000]`, and reaches 900,000 ms back. #[test] fn a_raw_data_window_ends_its_offset_before_the_evaluation() { - let window = asap_types::ir::materialized_view::RawDataWindow::new( + let window = asap_types::plan::materialized_view::RawDataWindow::new( Source::TimeSeries { metric: "latency".into(), }, @@ -543,162 +270,15 @@ fn a_raw_data_window_ends_its_offset_before_the_evaluation() { assert_eq!(asap_types::test_support::lookback_ms(&window), 900_000); } -/// `TimeRange(Range, width_ms)` over `shift` (if any) over `scan`, untimed. -fn window_over( - scan: &Rc, - width_ms: u64, - shift: Option, -) -> Rc { - let child = match shift { - Some(shift) => OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift, - child: Rc::clone(scan), - })) - .unwrap(), - None => Rc::clone(scan), - }; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(width_ms), - kind: asap_types::ir::TimeRangeKind::Range, - child, - })) - .unwrap() -} - -/// The exported DAG of `roots`, all at query time, and its one `Scan`'s id. -fn exported_with_scan( - roots: Vec>, -) -> ( - asap_types::ir::physical_export::PhysicalASAPDAG, - asap_types::ir::physical_export::PhysicalASAPNodeId, -) { - // One timing cache, so the roots keep sharing their `Scan`. - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = roots - .into_iter() - .map(|root| { - apply_materialization_timings(&root, &MaterializationAssignment::default(), &mut cache) - .unwrap() - }) - .map(QueryRoot::Operator) - .collect(); - let dag = compile_physical_asap_workload(&timed).unwrap(); - let scans: Vec<_> = dag - .nodes - .iter() - .filter(|node| matches!(node.payload, Operator::NonASAP(NonASAPOp::Scan { .. }))) - .map(|node| node.id) - .collect(); - assert_eq!(scans.len(), 1, "one shared Scan"); - (dag, scans[0]) -} - -/// A time-series `Scan` of `m` with a time index. -fn series_scan() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -// A `Scan` read by `m[5m]` and by `m[1h] offset 10m` is read over `(T - 70m, -// T]`, the union of `(T - 5m, T]` and `(T - 70m, T - 10m]`. -#[test] -fn a_scan_window_is_the_union_of_its_readers_windows() { - use asap_types::ir::physical_export::ScanWindow; - let minute_ms = 60_000; - let scan = series_scan(); - let offset = asap_types::ir::operator::TimeShift { - offset_ms: 10 * minute_ms as i64, - at: None, - }; - let (dag, scan_id) = exported_with_scan(vec![ - window_over(&scan, 5 * minute_ms, None), - window_over(&scan, 60 * minute_ms, Some(offset)), - ]); - let window = dag.scan_window(scan_id, &Default::default()); - assert_eq!( - window, - ScanWindow::Within { - end_offset_ms: 0, - width_ms: 70 * minute_ms as i64, - } - ); - assert_eq!(window.lookback_ms(), Some(70 * minute_ms)); - assert_eq!( - window.bounds_ms(100 * minute_ms as i64), - Some((30 * 60_000, 100 * 60_000)) - ); -} - -// A node above `m[5m]` and `m[1h] offset 10m` summarizes `(T - 70m, T]`, -// the union of the windows below it; a node with no window below it, the -// `Scan` itself, summarizes an unbounded window. -#[test] -fn a_view_window_is_the_union_of_the_windows_below_it() { - use asap_types::ir::physical_export::ScanWindow; - let minute_ms = 60_000; - let scan = series_scan(); - let offset = asap_types::ir::operator::TimeShift { - offset_ms: 10 * minute_ms as i64, - at: None, - }; - let both = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Concat { - children: vec![ - window_over(&scan, 5 * minute_ms, None), - window_over(&scan, 60 * minute_ms, Some(offset)), - ], - })) - .unwrap(); - let (dag, scan_id) = exported_with_scan(vec![both]); - assert_eq!( - dag.view_window(dag.roots[0]), - ScanWindow::Within { - end_offset_ms: 0, - width_ms: 70 * minute_ms as i64, - } - ); - assert_eq!(dag.view_window(scan_id), ScanWindow::Unbounded); -} - -// A window under an `@` anchor is not a fixed span back from the evaluation -// time, so the `Scan` it reads is read over an unbounded window. -#[test] -fn a_scan_read_under_an_at_anchor_has_an_unbounded_window() { - use asap_types::ir::physical_export::ScanWindow; - let scan = series_scan(); - let anchored = asap_types::ir::operator::TimeShift { - offset_ms: 0, - at: Some(asap_types::ir::operator::AtModifier::Timestamp(1_000)), - }; - let (dag, scan_id) = exported_with_scan(vec![ - window_over(&scan, 300_000, None), - window_over(&scan, 300_000, Some(anchored)), - ]); - assert_eq!( - dag.scan_window(scan_id, &Default::default()), - ScanWindow::Unbounded - ); -} - /// High-fan-in merge/concat validation is independent of wire edge order and /// preserves repeated input occurrences, rather than treating edges as a set. #[test] fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { - use asap_types::ir::physical_export::{ + use asap_types::ir::properties::ExecutionDataState; + use asap_types::plan::physical_asap_dag::{ EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, WindowEdgeCompatibility, }; - use asap_types::ir::properties::ExecutionDataState; let schema = Schema::new(vec![Field::plain("value", DataType::Float64, false)]); let count = 20_000; let mut nodes: Vec<_> = (0..count) @@ -712,10 +292,12 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { schema: schema.clone(), }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }) .collect(); let inputs: Vec<_> = (0..count).chain([0, 0]).collect(); @@ -725,10 +307,12 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { children: inputs.clone(), }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }); let mut dag = PhysicalASAPDAG { nodes, @@ -764,16 +348,3 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { Err(PhysicalASAPDAGValidationError::EdgeRoleMismatch { .. }) )); } - -/// A workload export of one timed root is that root's export, and an -/// untimed root is invalid. -#[test] -fn workload_export_matches_the_single_root_export() { - use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; - let timed = timed_at_ingestion(); - let roots = [QueryRoot::Operator(Rc::clone(&timed))]; - let exported = compile_physical_asap_workload_with_node_ids(&roots).unwrap(); - assert_eq!(exported.dag, compile_physical_asap_dag(&timed).unwrap()); - let untimed = [QueryRoot::Operator(plan())]; - assert!(compile_physical_asap_workload_with_node_ids(&untimed).is_err()); -} diff --git a/crates/types/tests/planner_vocabulary.rs b/crates/types/tests/planner_vocabulary.rs index 5f5a6ab7a..8a0c14a38 100644 --- a/crates/types/tests/planner_vocabulary.rs +++ b/crates/types/tests/planner_vocabulary.rs @@ -1,4 +1,4 @@ -use asap_types::ir::physical_export::WindowEdgeCompatibility; +use asap_types::plan::physical_asap_dag::WindowEdgeCompatibility; // The renamed window-edge compatibility still reads and emits the deployed // wire value. (Tumbling-window coverage names are checked in physical-asap-optimization.) diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index 6e61144a8..0f9024c32 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -1,9 +1,9 @@ +use asap_types::accuracy::AccuracyGuarantee; use asap_types::ir::operator::{AggIntent, Reduction, Source}; -use asap_types::ir::properties::{AccuracyGuarantee, ExecutionTiming}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; +use asap_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; use std::rc::Rc; fn scan(key_type: DataType, name: &str) -> Rc { @@ -23,10 +23,9 @@ fn aggregate(child: Rc, asap: bool) -> Rc { let operator = if asap { Operator::ASAP(ASAPOp::SummaryAgg { child, - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), input: SummaryUpdate::column(ColumnRef::Named("value".into())), reduction: Reduction::by(vec![0]), - grouping: GroupingStrategy::default(), filter: None, }) } else { @@ -67,8 +66,7 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { schema.fields[0].name = "alias".into(); schema.fields[0].table = Some("result".into()); let renamed = OperatorNode::with_schema(original.operator.clone(), schema) - .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))) - .with_timing(Some(ExecutionTiming::QueryTime)); + .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))); let original = Rc::new(renamed); original.validate_structure().unwrap(); let replacement = scan(DataType::Utf8, "new_key"); @@ -80,7 +78,6 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { Some(&DataType::Utf8) ); assert!(rebuilt.guarantee.is_none()); - assert!(rebuilt.timing.is_none()); rebuilt.validate_structure().unwrap(); } } @@ -229,50 +226,6 @@ fn series_selector() -> Rc { .unwrap() } -/// `topk(5, a)` read from a current-series population of `a`, built by hand. -fn current_series_topk() -> Rc { - use asap_types::ir::operator::maintained_population::*; - let selector = series_selector(); - let population = OperatorNode::new_shared(Operator::ASAP(ASAPOp::MaintainPopulation { - child: selector, - population: MaintainedPopulation { - input: PopulationInput::CurrentSeries(CurrentSeriesInput { - metric: "a".into(), - matchers: vec![], - grouping: vec![], - without: false, - lookback_ms: 1_000, - }), - max_k: 5, - quantiles: false, - }, - })) - .unwrap(); - OperatorNode::new_shared(Operator::ASAP(ASAPOp::EvaluatePopulation { - child: population, - evaluation: PopulationStatistic::TopK { k: 5 }, - })) - .unwrap() -} - -/// Time `root` under the default materialization assignment, which runs the -/// data-state and population-contract validation, and export it. -fn compile_physical_asap_dag(root: &Rc) -> Result<(), String> { - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - root.validate_structure().map_err(|e| e.to_string())?; - let timed = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .map_err(|e| format!("{e:?}"))?; - asap_types::ir::physical_export::compile_physical_asap_dag(&timed) - .map_err(|e| format!("{e:?}"))?; - Ok(()) -} - /// A table-row population maintains a numeric value column: one that names /// a label column fails validation. #[test] @@ -331,80 +284,3 @@ fn a_current_series_population_matches_only_its_selectors_lookback() { }; assert!(!asap_types::test_support::matches_node(&wrong, &selector)); } - -/// Population timing is a materialization choice: the population follows its -/// assignment (`NotMaterialized` by default), while its evaluation always -/// runs at query time. -#[test] -fn population_timing_follows_its_materialization() { - use asap_types::ir::physical_export::compile_physical_asap_dag as export_timed; - use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - use asap_types::physical::{Materialization, Medium}; - let candidate = current_series_topk(); - let Some(ASAPOp::EvaluatePopulation { - child: population, .. - }) = candidate.asap() - else { - unreachable!() - }; - let on_disk = Materialization::MatAtIngestion(Medium::Disk); - let mut at_ingestion = MaterializationAssignment::default(); - at_ingestion.set(population, on_disk); - for (assignment, timing, materialization) in [ - ( - MaterializationAssignment::default(), - QueryTime, - Materialization::NotMaterialized, - ), - (at_ingestion, IngestionTime, on_disk), - ] { - let timed = apply_materialization_timings( - &candidate, - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = export_timed(&timed).unwrap(); - let [population, evaluation] = &dag.nodes[dag.nodes.len() - 2..] else { - unreachable!() - }; - assert_eq!(population.output_state.timing, timing); - assert_eq!(population.materialization, materialization); - assert_eq!(evaluation.output_state.timing, QueryTime); - } - let mut evaluation_at_ingestion = (*candidate).clone(); - evaluation_at_ingestion.timing = Some(IngestionTime); - assert!(compile_physical_asap_dag(&Rc::new(evaluation_at_ingestion)).is_err()); -} - -/// An evaluation cannot exceed its population's contract (`max_k`), and a -/// population cannot claim a source it does not read. -#[test] -fn malformed_population_dags_fail_closed() { - use asap_types::ir::operator::maintained_population::{PopulationInput, PopulationStatistic}; - let candidate = current_series_topk(); - compile_physical_asap_dag(&candidate).expect("the unmodified candidate is legal"); - let mut bad = (*candidate).clone(); - let Operator::ASAP(ASAPOp::EvaluatePopulation { evaluation, .. }) = &mut bad.operator else { - unreachable!() - }; - *evaluation = PopulationStatistic::TopK { k: 6 }; - assert!(compile_physical_asap_dag(&Rc::new(bad.clone())).is_err()); - let Operator::ASAP(ASAPOp::EvaluatePopulation { child, evaluation }) = &mut bad.operator else { - unreachable!() - }; - *evaluation = PopulationStatistic::TopK { k: 5 }; - let producer = Rc::make_mut(child); - let Operator::ASAP(ASAPOp::MaintainPopulation { population, .. }) = &mut producer.operator - else { - unreachable!() - }; - let PopulationInput::CurrentSeries(spec) = &mut population.input else { - unreachable!() - }; - spec.metric = "b".into(); - assert!(compile_physical_asap_dag(&Rc::new(bad)).is_err()); -} diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index 0c16614cf..415010c89 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -89,28 +89,16 @@ fn scalar_type_rules_fail_closed() { fn state_evaluations_and_passthrough_keep_their_contracts() { use asap_types::ir::operator::Reduction; use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, - }; + use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator, ProjectItem}; - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 100, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ); + use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let state = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: scan(), - family, + summary_type: family, input: SummaryUpdate::column(ColumnRef::Named("x".into())), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap(), @@ -127,7 +115,9 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { .unwrap(); pass.validate_structure().unwrap(); assert_eq!(pass.result_kind, OperatorResultKind::State); - assert!(ScalarExpr::Column(0).scalar_type(&pass.schema).is_err()); + assert!(ScalarExpr::>::Column(0) + .scalar_type(&pass.schema) + .is_err()); assert!(validate_asap_inputs(&ASAPOp::SummaryEstimate { summary_input: state.clone(), query: SketchStatistic::Cardinality @@ -141,53 +131,22 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { assert!(validate_asap_inputs(&ASAPOp::FinalizeExactAccumulator { child: state }).is_err()); } -/// Phase validation checks dependencies, without declaring a computation query-only. -#[test] -fn execution_timing_checks_edges_not_function_names() { - use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; - use asap_types::ir::ProjectItem; - let input = Rc::new((*scan()).clone().with_timing(Some(IngestionTime))); - let mut project = OperatorNode::new(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - child: input, - cols: vec![ProjectItem { - expr: ScalarExpr::FunctionCall { - name: "promql_abs".into(), - args: vec![ScalarExpr::Column(0)], - }, - alias: None, - }], - qualifier: None, - })) - .unwrap() - .with_timing(Some(IngestionTime)); - Rc::new(project.clone()) - .validate_execution_timing() - .unwrap(); - if let asap_types::ir::Operator::NonASAP(NonASAPOp::Project { child, .. }) = - &mut project.operator - { - *child = Rc::new(child.as_ref().clone().with_timing(Some(QueryTime))); - } - assert!(Rc::new(project).validate_execution_timing().is_err()); -} - /// Both operator categories use the same fallible schema-deriving constructor. #[test] fn shared_construction_derives_both_operator_categories() { use asap_types::ir::operator::Reduction; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::FieldDataType; - use asap_types::ir::schema::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; use asap_types::ir::{ASAPOp, Operator, ProjectItem}; + use asap_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; let input = scan(); let state = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: input.clone(), - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), input: SummaryUpdate::column(ColumnRef::Named("x".into())), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap(), @@ -195,7 +154,6 @@ fn shared_construction_derives_both_operator_categories() { assert_eq!(state.result_kind, OperatorResultKind::State); assert!(!state.schema.fields.last().unwrap().is_plain()); assert!(state.guarantee.is_none()); - assert!(state.timing.is_none()); state.validate_structure().unwrap(); assert!( diff --git a/crates/types/tests/summary_coverage.rs b/crates/types/tests/summary_coverage.rs deleted file mode 100644 index 07357264b..000000000 --- a/crates/types/tests/summary_coverage.rs +++ /dev/null @@ -1,1096 +0,0 @@ -//! Summary coverage is `definition + selection`, derived from the sub-DAG -//! (`docs/design_docs/proposals/asap-primitive-schema.md` §4.2). Merges need -//! the same definition and disjoint selections. -use std::ops::Bound; -use std::rc::Rc; -use std::time::Duration; - -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::operator::TimeShift; -use asap_types::ir::properties::summary_coverage::{ - ColumnIdentity, Constraint, CoverageError, SelectionBox, -}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{ - ExactKind, ExactParams, GroupingStrategy, MergeInputRule, SketchAlgorithm, SketchKind, - SketchParams, SummaryUpdate, -}; -use asap_types::ir::{ - ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, Predicate, - ProjectItem, ScalarExpr, SchemaDerivationError, TimeRangeKind, -}; -use asap_types::test_support::definition_id; - -const JOB: usize = 0; -const REGION: usize = 1; -const LATENCY: usize = 2; - -fn table() -> Rc { - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - Field::plain("size", DataType::Float64, false), - ]) -} - -fn table_with(fields: Vec) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema::new(fields), - })) - .unwrap() -} - -/// PromQL leaf `m{predicates}` with columns `[ts, value, job]`. -fn series(predicates: Vec) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates, - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -fn compare(column: usize, op: CompareOpKind, value: ScalarValue) -> ScalarExpr { - ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(column)), - op, - right: Box::new(ScalarExpr::Literal(value)), - semantics: ExprSemantics::Sql, - } -} - -fn eq(column: usize, value: &str) -> ScalarExpr { - compare(column, CompareOpKind::Eq, ScalarValue::Utf8(value.into())) -} - -fn filter(child: Rc, pred: ScalarExpr) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(pred), - child, - })) - .unwrap() -} - -fn kll(child: Rc, column: &str) -> Rc { - summary( - child, - column, - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), - ) -} - -fn summary(child: Rc, column: &str, family: FieldDataType) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input: SummaryUpdate::column(ColumnRef::Named(column.into())), - reduction: Reduction::by(vec![JOB]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() -} - -/// KLL of latency by job over `t` restricted by `pred`. -fn latency_where(pred: ScalarExpr) -> Rc { - kll(filter(table(), pred), "latency") -} - -/// Tumbling window: `TimeRange(1m)` over `TimeShift(shift)` over `m`. -fn tumbling_window(shift_ms: i64, predicates: Vec) -> Rc { - kll(tumbling_window_rows(shift_ms, predicates), "value") -} - -/// Exact counter-increase tumbling window over the same rows as [`tumbling_window`]. -fn increase_tumbling_window(shift_ms: i64, predicates: Vec) -> Rc { - summary( - tumbling_window_rows(shift_ms, predicates), - "value", - FieldDataType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - ) -} - -/// KLL over `TimeRange(width_ms)` over `TimeShift(shift_ms)` over `m`: the -/// window `(-(shift + width), -shift]`. -fn window(shift_ms: i64, width_ms: u64) -> Rc { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift_ms, - at: None, - }, - child: series(vec![]), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(width_ms), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap(); - kll(range, "value") -} - -fn tumbling_window_rows(shift_ms: i64, predicates: Vec) -> Rc { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift_ms, - at: None, - }, - child: series(predicates), - })) - .unwrap(); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(60_000), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap() -} - -fn merge(children: Vec>) -> Result, SchemaDerivationError> { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: None, - group_by: None, - })) -} - -fn rejected(children: Vec>) -> CoverageError { - match merge(children) { - Err(SchemaDerivationError::Coverage(error)) => error, - other => panic!("expected a coverage error, got {other:?}"), - } -} - -fn column(name: &str) -> ColumnIdentity { - ColumnIdentity::new(None, name) -} - -fn utf8(values: &[&str]) -> Constraint { - Constraint::In( - values - .iter() - .map(|v| ScalarValue::Utf8((*v).into())) - .collect(), - ) -} - -fn relative(from_ms: i64, to_ms: i64) -> Option<(Bound, Bound)> { - Some((Bound::Excluded(from_ms), Bound::Included(to_ms))) -} - -/// A lifted filter leaves the definition over the bare scan. -#[test] -fn filters_lift_into_the_selection() { - let scan = table(); - let us = kll(filter(Rc::clone(&scan), eq(REGION, "us")), "latency"); - let coverage = us.coverage().unwrap(); - assert!(Rc::ptr_eq(coverage.definition().children()[0], &scan)); - assert_eq!( - coverage.selection(), - vec![SelectionBox::new( - [(column("region"), utf8(&["us"]))].into(), - None - )] - ); -} - -/// Different populations merge into one box; the same population twice -/// would count every row twice. -#[test] -fn populations_merge_when_disjoint() { - let merged = merge(vec![ - latency_where(eq(REGION, "us")), - latency_where(eq(REGION, "eu")), - ]) - .unwrap(); - assert_eq!( - merged.coverage().unwrap().selection()[0].columns()[&column("region")], - utf8(&["us", "eu"]) - ); - assert_eq!( - rejected(vec![ - latency_where(eq(REGION, "us")), - latency_where(eq(REGION, "us")), - ]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Equal schemas do not make a KLL over latency and one over size mergeable. -#[test] -fn different_inputs_do_not_merge() { - assert_eq!( - rejected(vec![ - kll(filter(table(), eq(REGION, "us")), "latency"), - kll(filter(table(), eq(REGION, "eu")), "size"), - ]), - CoverageError::DefinitionMismatch - ); -} - -/// The same update over scans of different columns is a different computation. -#[test] -fn the_same_update_over_different_scan_schemas_does_not_merge() { - let scan = |measure: &str| { - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain(measure, DataType::Float64, false), - Field::plain("value", DataType::Float64, false), - ]) - }; - assert_eq!( - rejected(vec![ - kll(filter(scan("latency"), eq(REGION, "us")), "value"), - kll(filter(scan("size"), eq(REGION, "eu")), "value"), - ]), - CoverageError::DefinitionMismatch - ); -} - -/// `shipping.region` and `billing.region` are different columns, so -/// restricting each says nothing about overlap. -#[test] -fn qualified_columns_stay_distinct() { - let scan = || { - let qualified = |table: &str| Field { - table: Some(table.into()), - ..Field::plain("region", DataType::Utf8, false) - }; - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - qualified("shipping"), - qualified("billing"), - Field::plain("latency", DataType::Float64, false), - ]) - }; - assert_eq!( - rejected(vec![ - kll(filter(scan(), eq(1, "us")), "latency"), - kll(filter(scan(), eq(2, "eu")), "latency"), - ]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Tumbling windows take their time from `TimeRange` over `TimeShift`: adjacent tumbling windows -/// join, a gap stays two boxes, and the same tumbling window twice overlaps. -#[test] -fn time_tumbling_windows_merge_and_keep_gaps() { - let first = tumbling_window(0, vec![]); - assert_eq!( - first.coverage().unwrap().selection()[0].relative_time(), - relative(-60_000, 0) - ); - assert!(Rc::ptr_eq( - first.coverage().unwrap().definition().children()[0], - first.children()[0].children()[0].children()[0] - )); - let joined = merge(vec![first.clone(), tumbling_window(60_000, vec![])]).unwrap(); - assert_eq!( - joined.coverage().unwrap().selection(), - vec![SelectionBox::new(Default::default(), relative(-120_000, 0))] - ); - let gapped = merge(vec![first.clone(), tumbling_window(120_000, vec![])]).unwrap(); - assert_eq!(gapped.coverage().unwrap().selection().len(), 2); - assert_eq!( - rejected(vec![first.clone(), first]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Counter tumbling-window merge only when their time windows are disjoint: states -/// disjoint by a label but over the same window would interleave in time. -#[test] -fn counter_tumbling_windows_merge_only_disjoint_in_time() { - let joined = merge(vec![ - increase_tumbling_window(0, vec![]), - increase_tumbling_window(60_000, vec![]), - ]) - .unwrap(); - assert_eq!( - joined.coverage().unwrap().selection()[0].relative_time(), - relative(-120_000, 0) - ); - let by_job = |job| increase_tumbling_window(0, vec![Predicate(eq(2, job))]); - assert_eq!( - rejected(vec![by_job("api"), by_job("web")]), - CoverageError::InputsOverlap { - required: MergeInputRule::DisjointInTime, - } - ); -} - -/// A day of one-minute tumbling windows (1,440 inputs) merges into one window, in -/// either order: the inputs' windows are sorted and only neighbours are -/// compared, so the check is O(n log n), not all pairs. A tumbling window listed -/// twice still overlaps. -#[test] -fn a_day_of_minute_tumbling_windows_merges_into_one_window() { - const TUMBLING_WINDOW_COUNT: i64 = 1_440; - let day = relative(-TUMBLING_WINDOW_COUNT * 60_000, 0); - let newest_first: Vec<_> = (0..TUMBLING_WINDOW_COUNT) - .map(|i| tumbling_window(i * 60_000, vec![])) - .collect(); - let oldest_first: Vec<_> = newest_first.iter().rev().cloned().collect(); - for tumbling_windows in [newest_first.clone(), oldest_first] { - let merged = merge(tumbling_windows).unwrap(); - merged.validate_structure().unwrap(); - let coverage = merged.coverage().unwrap(); - let selection = coverage.selection(); - assert_eq!(selection.len(), 1); - assert_eq!(selection[0].relative_time(), day); - } - let counters: Vec<_> = (0..TUMBLING_WINDOW_COUNT) - .map(|i| increase_tumbling_window(i * 60_000, vec![])) - .collect(); - assert_eq!( - merge(counters).unwrap().coverage().unwrap().selection()[0].relative_time(), - day - ); - let mut repeated = newest_first; - repeated.push(tumbling_window(30 * 60_000, vec![])); - assert_eq!( - rejected(repeated), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// A long window that spans two short ones overlaps both, though the -/// short ones are apart: sorted by start, the long window's neighbour -/// overlaps it, so the all-pairs check runs and rejects the merge. The -/// short windows alone merge as two boxes. -#[test] -fn a_long_window_spanning_others_is_rejected() { - let long = window(0, 100_000); - let (near, far) = (window(10_000, 10_000), window(50_000, 10_000)); - assert_eq!( - rejected(vec![long.clone(), near.clone(), far.clone()]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); - assert_eq!( - rejected(vec![near.clone(), far.clone(), long]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); - let apart = merge(vec![far, near]).unwrap(); - assert_eq!( - apart - .coverage() - .unwrap() - .selection() - .iter() - .map(|b| b.relative_time()) - .collect::>(), - [relative(-60_000, -50_000), relative(-20_000, -10_000)] - ); -} - -/// Label matchers on the scan lift like any filter. -#[test] -fn scan_predicates_lift() { - let api = tumbling_window(0, vec![Predicate(eq(2, "api"))]); - let coverage = api.coverage().unwrap(); - assert_eq!( - coverage.selection()[0].columns()[&column("job")], - utf8(&["api"]) - ); - let scan = coverage.definition().children()[0]; - assert!(matches!( - scan.non_asap(), - Some(NonASAPOp::Scan { predicates, .. }) if predicates.is_empty() - )); - merge(vec![api, tumbling_window(0, vec![Predicate(eq(2, "web"))])]).unwrap(); -} - -/// Value ranges are selections: `latency < 100` and `latency >= 100` are -/// disjoint, `latency <= 100` overlaps `latency >= 100`. -#[test] -fn value_ranges_merge_when_disjoint() { - let latency = |op| latency_where(compare(LATENCY, op, ScalarValue::Float64(100.0))); - merge(vec![latency(CompareOpKind::Lt), latency(CompareOpKind::Ge)]).unwrap(); - assert_eq!( - rejected(vec![latency(CompareOpKind::Le), latency(CompareOpKind::Ge)]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Touching value ranges join like touching time windows; the joined range -/// stays explicit, since neither input took NULL rows. A gap stays apart. -#[test] -fn touching_value_ranges_join() { - let latency = |op, v| latency_where(compare(LATENCY, op, ScalarValue::Float64(v))); - let joined = merge(vec![ - latency(CompareOpKind::Lt, 100.0), - latency(CompareOpKind::Ge, 100.0), - ]) - .unwrap(); - assert_eq!( - joined.coverage().unwrap().selection(), - vec![SelectionBox::new( - [( - column("latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Unbounded, - }, - )] - .into(), - None - )] - ); - let gap = merge(vec![ - latency(CompareOpKind::Lt, 100.0), - latency(CompareOpKind::Gt, 100.0), - ]) - .unwrap(); - assert_eq!(gap.coverage().unwrap().selection().len(), 2); -} - -/// A predicate that is not a value set or interval on one column stays in -/// the definition, so states with different residuals do not merge. -#[test] -fn residual_predicates_stay_in_the_definition() { - let doubled_above = |threshold: f64, region: &str| { - let doubled = ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(ScalarExpr::Column(LATENCY)), - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), - semantics: ExprSemantics::Sql, - }; - latency_where(ScalarExpr::BoolAnd(vec![ - ScalarExpr::Compare { - left: Box::new(doubled), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(threshold))), - semantics: ExprSemantics::Sql, - }, - eq(REGION, region), - ])) - }; - let state = doubled_above(10.0, "us"); - let coverage = state.coverage().unwrap(); - let definition = coverage.definition(); - assert!(matches!( - definition.children()[0].non_asap(), - Some(NonASAPOp::Filter { .. }) - )); - merge(vec![state.clone(), doubled_above(10.0, "eu")]).unwrap(); - assert_eq!( - rejected(vec![state, doubled_above(20.0, "eu")]), - CoverageError::DefinitionMismatch - ); -} - -/// An instant selector picks the latest sample per series, which is not a -/// selection of rows, so it stays in the definition. -#[test] -fn instant_selectors_stay_in_the_definition() { - let instant = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(300_000), - kind: TimeRangeKind::Instant, - child: series(vec![]), - })) - .unwrap(); - let state = kll(Rc::clone(&instant), "value"); - let coverage = state.coverage().unwrap(); - assert_eq!(coverage.selection()[0].relative_time(), None); - assert!(Rc::ptr_eq(coverage.definition().children()[0], &instant)); -} - -/// A merge is a state with coverage, so merges nest. -#[test] -fn merges_nest() { - let us_eu = merge(vec![ - latency_where(eq(REGION, "us")), - latency_where(eq(REGION, "eu")), - ]) - .unwrap(); - merge(vec![us_eu.clone(), latency_where(eq(REGION, "apac"))]).unwrap(); - assert_eq!( - rejected(vec![us_eu, latency_where(eq(REGION, "eu"))]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// When a state is computed is not what it computes. -#[test] -fn timing_does_not_block_merges() { - let ingested = Rc::new( - (*tumbling_window(60_000, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)), - ); - let queried = Rc::new( - (*tumbling_window(0, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - merge(vec![ingested, queried]).unwrap(); -} - -/// A merge built around `new` is still checked by `validate_structure`. -#[test] -fn forged_merges_fail_validation() { - let us = latency_where(eq(REGION, "us")); - let schema = us.schema.clone(); - let forged = Rc::new(OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![us.clone(), us], - newest_tumbling_windows: None, - group_by: None, - }), - schema, - )); - assert!(forged.coverage().is_none()); - assert!(matches!( - forged.validate_structure(), - Err(SchemaDerivationError::Coverage( - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - )) - )); -} - -/// Only summary states have coverage. -#[test] -fn non_summary_nodes_have_no_coverage() { - assert!(table().coverage().is_none()); - assert!(filter(table(), eq(REGION, "us")).coverage().is_none()); -} - -/// Two output columns with the same `(table, name)` cannot be told apart in -/// a selection, so restrictions on them stay in the definition. -#[test] -fn ambiguous_column_names_do_not_lift() { - let renamed = |restricted: usize, value: &str| { - let item = |column: usize, alias: Option<&str>| ProjectItem { - alias: alias.map(Into::into), - expr: ScalarExpr::Column(column), - }; - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: vec![ - item(JOB, None), - item(REGION, Some("k")), - item(LATENCY, Some("k")), - item(LATENCY, None), - ], - qualifier: None, - child: table(), - })) - .unwrap(); - kll(filter(project, eq(restricted, value)), "latency") - }; - assert_eq!( - rejected(vec![renamed(1, "us"), renamed(2, "eu")]), - CoverageError::DefinitionMismatch - ); -} - -/// A rename outside a `Project` stays in the definition: here B's `Filter` -/// swaps the names of `region` and `tier`, so B's `tier = 'eu'` must not -/// read as `region = 'eu'` and look disjoint from A's `region = 'us'`. -#[test] -fn renaming_filters_stay_in_the_definition() { - let scan = || { - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("tier", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - ]) - }; - let us = kll(filter(scan(), eq(1, "us")), "latency"); - let swapped = filter(scan(), eq(2, "eu")); - let mut schema = swapped.schema.clone(); - schema.fields[1].name = "tier".into(); - schema.fields[2].name = "region".into(); - let swapped = OperatorNode::with_schema(swapped.operator.clone(), schema); - let eu_tier = kll(Rc::new(swapped), "latency"); - assert_eq!( - rejected(vec![us, eu_tier]), - CoverageError::DefinitionMismatch - ); -} - -/// `latency = 1` and `latency = 1.0` select the same rows of a Float64 column. -#[test] -fn literals_of_different_types_are_not_proven_different() { - let one = |value| latency_where(compare(LATENCY, CompareOpKind::Eq, value)); - assert!(merge(vec![ - one(ScalarValue::Int64(1)), - one(ScalarValue::Float64(1.0)) - ]) - .is_err()); -} - -/// A scan predicate whose conjuncts only partly lift keeps just the rest. -#[test] -fn partly_lifted_scan_predicates_keep_only_the_rest() { - let doubled_positive = ScalarExpr::Compare { - left: Box::new(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(ScalarExpr::Column(1)), - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), - semantics: ExprSemantics::Promql, - }), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(0.0))), - semantics: ExprSemantics::Promql, - }; - let job = |name: &str| { - tumbling_window( - 0, - vec![Predicate(ScalarExpr::BoolAnd(vec![ - eq(2, name), - doubled_positive.clone(), - ]))], - ) - }; - merge(vec![job("api"), job("web")]).unwrap(); -} - -/// The worked example of the design doc (§4.2.2), as the SQL frontend -/// lowers it: the `FILTER` condition and `region = 'us'` (in -/// `Scan.predicates`, through a renaming `Project`) move into the -/// selection; the condition on an expression stays. -#[test] -fn design_doc_worked_example() { - let doubled_above_10 = ScalarExpr::Compare { - left: Box::new(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(ScalarExpr::Column(LATENCY)), - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(2))), - semantics: ExprSemantics::Sql, - }), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(10))), - semantics: ExprSemantics::Sql, - }; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![Predicate(ScalarExpr::BoolAnd(vec![ - eq(REGION, "us"), - doubled_above_10.clone(), - ]))], - schema: Schema::new(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - ]), - })) - .unwrap(); - let item = |column: usize, alias: Option<&str>| ProjectItem { - alias: alias.map(Into::into), - expr: ScalarExpr::Column(column), - }; - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: vec![ - item(JOB, None), - item(REGION, Some("r")), - item(LATENCY, None), - ], - qualifier: None, - child: scan, - })) - .unwrap(); - let below_100 = compare(2, CompareOpKind::Lt, ScalarValue::Int64(100)); - let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: project, - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), - input: SummaryUpdate::column(ColumnRef::Named("latency".into())), - reduction: Reduction::by(vec![0]), - grouping: GroupingStrategy::default(), - filter: Some(Predicate(below_100)), - })) - .unwrap(); - - let coverage = state.coverage().unwrap(); - assert_eq!( - coverage.selection(), - vec![SelectionBox::new( - [ - (column("r"), utf8(&["us"])), - ( - column("latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Excluded(ScalarValue::Int64(100)), - }, - ), - ] - .into(), - None - )] - ); - // definition: SummaryAgg (no filter) over Project over - // Scan t with only `latency * 2 > 10` left in its predicates. - let Some(ASAPOp::SummaryAgg { filter, child, .. }) = coverage.definition().asap() else { - panic!("definition is a SummaryAgg"); - }; - assert!(filter.is_none()); - assert!(matches!(child.non_asap(), Some(NonASAPOp::Project { .. }))); - assert!(matches!( - child.children()[0].non_asap(), - Some(NonASAPOp::Scan { predicates, .. }) if *predicates == vec![Predicate(doubled_above_10)] - )); -} - -/// The coverage of `node`, which must have one. -fn coverage_of( - node: &OperatorNode, -) -> asap_types::ir::properties::summary_coverage::SummaryCoverage { - node.coverage().expect("a summary state") -} - -/// Equal definitions get equal ids whatever the node, its selection, timing -/// or guarantee; a different computation (another metric, another column, -/// another sketch) gets another id. -#[test] -fn equal_definitions_get_equal_ids() { - let id = |node: &OperatorNode| definition_id(&coverage_of(node)); - let us = latency_where(eq(REGION, "us")); - let eu = latency_where(eq(REGION, "eu")); - assert_eq!(id(&us), id(&eu)); - let ingested = (*tumbling_window(0, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)); - assert_eq!(id(&ingested), id(&tumbling_window(60_000, vec![]))); - assert_eq!(id(&ingested).len(), 32); - let size = kll(filter(table(), eq(REGION, "eu")), "size"); - assert_ne!(id(&size), id(&eu)); - assert_ne!( - id(&tumbling_window(0, vec![])), - id(&increase_tumbling_window(0, vec![])) - ); - assert_ne!(id(&tumbling_window(0, vec![])), id(&us)); -} - -/// Six 10-minute tumbling windows at offsets 0 to 50 minutes share one definition id, -/// each with its own time selection `(-(i + 1) · 10m, -i · 10m]`; their -/// merge has the same id and the hour as its selection. -#[test] -fn tumbling_windows_of_an_hour_share_one_definition_with_distinct_times() { - let tumbling_windows: Vec> = - (0..6).map(|i| window(i * 600_000, 600_000)).collect(); - let coverages: Vec<_> = tumbling_windows - .iter() - .map(|tumbling_window| coverage_of(tumbling_window)) - .collect(); - for (i, coverage) in coverages.iter().enumerate() { - let i = i as i64; - assert_eq!(definition_id(coverage), definition_id(&coverages[0])); - assert_eq!( - coverage.selection(), - [SelectionBox::new( - Default::default(), - relative(-(i + 1) * 600_000, -i * 600_000) - )] - ); - } - let hour = coverage_of(&merge(tumbling_windows).unwrap()); - assert_eq!(definition_id(&hour), definition_id(&coverages[0])); - assert_eq!( - hour.selection(), - [SelectionBox::new( - Default::default(), - relative(-3_600_000, 0) - )] - ); -} - -/// A selection round-trips through JSON; columns are a list and bounds -/// name their kind. -#[test] -fn selections_round_trip_through_json() { - let selection = SelectionBox::new( - [ - (column("region"), utf8(&["eu", "us"])), - ( - column("latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Excluded(ScalarValue::Float64(100.0)), - }, - ), - ] - .into(), - relative(-120_000, -60_000), - ); - let json = serde_json::to_value(&selection).unwrap(); - assert_eq!( - json["relative_time"], - serde_json::json!([{"excluded": -120000}, {"included": -60000}]) - ); - assert_eq!(json["columns"].as_array().unwrap().len(), 2); - assert_eq!( - serde_json::from_value::(json).unwrap(), - selection - ); -} - -/// A `family` build over `child`'s `latency` (a table) or `value` (a -/// series), grouped by `reduction`. -fn grouped( - child: Rc, - family: FieldDataType, - reduction: Reduction, -) -> Rc { - let column = match child.schema.column_id("latency") { - Some(_) => "latency", - None => "value", - }; - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input: SummaryUpdate::column(ColumnRef::Named(column.into())), - reduction, - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() -} - -fn kll_family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) -} - -fn rollup(state: Rc, keys: Vec) -> Result, String> { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![state], - group_by: Some(keys), - newest_tumbling_windows: None, - })) - .map_err(|error| error.to_string()) -} - -/// States by (job, region) over `region ∈ {us, eu}`, rolled up to region, -/// cover what a direct build by region over those rows does: the same -/// definition (and id) and the same selection. -#[test] -fn a_rollup_covers_the_coarser_build() { - let rows = filter( - table(), - ScalarExpr::InList { - expr: Box::new(ScalarExpr::Column(REGION)), - list: vec![ - ScalarExpr::Literal(ScalarValue::Utf8("us".into())), - ScalarExpr::Literal(ScalarValue::Utf8("eu".into())), - ], - negated: false, - }, - ); - let finer = grouped(rows.clone(), kll_family(), Reduction::by(vec![JOB, REGION])); - let coarse = grouped(rows, kll_family(), Reduction::by(vec![REGION])); - let rolled = rollup(finer, vec![1]).unwrap(); - assert_eq!(rolled.schema, coarse.schema); - let (rolled, coarse) = (rolled.coverage().unwrap(), coarse.coverage().unwrap()); - assert_eq!(definition_id(&rolled), definition_id(&coarse)); - assert_eq!(rolled.selection(), coarse.selection()); - assert_eq!( - rolled.selection()[0].columns()[&column("region")], - utf8(&["us", "eu"]) - ); -} - -/// Per-series states keep their input's columns in place, so a rollup by -/// the `job` label covers a build by `job` over the same series. -#[test] -fn a_rollup_of_per_series_states_covers_the_build_by_its_labels() { - let rows = tumbling_window_rows(0, vec![]); - let job = rows.schema.column_id("job").unwrap(); - let per_series = grouped(rows.clone(), kll_family(), Reduction::PerEntity); - let state_job = per_series.schema.column_id("job").unwrap(); - let rolled = rollup(per_series, vec![state_job]).unwrap(); - let by_job = grouped(rows, kll_family(), Reduction::by(vec![job])); - assert_eq!( - definition_id(&rolled.coverage().unwrap()), - definition_id(&by_job.coverage().unwrap()) - ); -} - -/// A rollup is rejected when a key is no group key of the input's build, -/// and when its family cannot merge different groups (counter states merge -/// consecutive windows of one series, IRate has no merge). -#[test] -fn invalid_rollups_fail_construction() { - let without = grouped( - table(), - kll_family(), - Reduction::Reduce( - asap_types::ir::operator::operator_properties::GroupKeys::without(vec![REGION]), - ), - ); - assert!(rollup(without, vec![0]) - .unwrap_err() - .contains(&CoverageError::NotAGroupKey { key: 0 }.to_string())); - for (kind, params) in [ - (ExactKind::Increase, ExactParams::Increase), - (ExactKind::Rate, ExactParams::Rate), - (ExactKind::IRate, ExactParams::IRate), - ] { - let state = grouped( - tumbling_window_rows(0, vec![]), - FieldDataType::ExactAggregate(kind, params), - Reduction::by(vec![2]), - ); - assert!( - rollup(state, vec![]) - .unwrap_err() - .contains(&CoverageError::FamilyCannotRegroup.to_string()), - "{kind:?}" - ); - } -} - -/// A rollup to no keys of a SQL relation's states covers the global build -/// over the same rows: a global SQL aggregate, one row even over no input -/// rows, as the rollup's one state. -#[test] -fn a_rollup_to_no_keys_of_a_relation_covers_the_global_build() { - let by_job = grouped(table(), kll_family(), Reduction::by(vec![JOB])); - let global = grouped(table(), kll_family(), Reduction::by(vec![])); - let rolled = rollup(by_job, vec![]).unwrap(); - assert_eq!(rolled.schema, global.schema); - assert_eq!( - definition_id(&rolled.coverage().unwrap()), - definition_id(&global.coverage().unwrap()) - ); -} - -/// `group_by` is left out of JSON when unset, so a merge that keeps its -/// groups serializes as before; a rollup's keys round-trip. -#[test] -fn group_by_is_serialized_only_for_a_rollup() { - let state = grouped(table(), kll_family(), Reduction::by(vec![JOB, REGION])); - let merge = ASAPOp::SummaryMerge { - children: vec![0u32], - group_by: None, - newest_tumbling_windows: None, - }; - assert_eq!( - serde_json::to_value(&merge).unwrap(), - serde_json::json!({"SummaryMerge": {"children": [0]}}) - ); - let rollup_op: ASAPOp = ASAPOp::SummaryMerge { - children: vec![0], - group_by: Some(vec![1]), - newest_tumbling_windows: None, - }; - let json = serde_json::to_string(&rollup_op).unwrap(); - assert_eq!( - serde_json::from_str::>(&json).unwrap(), - rollup_op - ); - assert!(rollup(state, vec![1]).is_ok()); -} - -/// An exact sum rolled up from (job, region) to region finalizes to the -/// direct build's column, type and nullability. -#[test] -fn a_finalized_rollup_has_the_direct_builds_schema() { - let sum = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let finer = grouped(table(), sum.clone(), Reduction::by(vec![JOB, REGION])); - let coarse = grouped(table(), sum, Reduction::by(vec![REGION])); - let finalize = |state: Rc| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: state, - })) - .unwrap() - }; - assert_eq!( - finalize(rollup(finer, vec![1]).unwrap()).schema, - finalize(coarse).schema - ); -} - -/// A filter on the group keys of per-(job, region) states keeps whole -/// states, so it is still state: merging the kept states covers the global -/// build over the filtered rows. A filter reading the state column is not a -/// selection of states and is rejected. -#[test] -fn a_key_filter_over_states_selects_states() { - let states = grouped(table(), kll_family(), Reduction::by(vec![JOB, REGION])); - let kept = filter(Rc::clone(&states), eq(JOB, "a")); - assert_eq!(kept.result_kind, OperatorResultKind::State); - let rolled = rollup(kept, vec![]).unwrap(); - let direct = grouped( - filter(table(), eq(JOB, "a")), - kll_family(), - Reduction::by(vec![]), - ); - assert_eq!(rolled.schema, direct.schema); - assert_eq!( - definition_id(&rolled.coverage().unwrap()), - definition_id(&direct.coverage().unwrap()) - ); - let on_state = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::IsNull(Box::new(ScalarExpr::Column(2)))), - child: states, - })) - .unwrap(); - assert_eq!(on_state.result_kind, OperatorResultKind::Relation); - assert!(on_state.validate_structure().is_err()); -} diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index b76ad5cc7..2c34c39c3 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -1,36 +1,21 @@ //! Window composition merges compatible summary states without consuming raw rows. -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::operator::arguments::{Reduction, Source}; use asap_types::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - DataType, ExactKind, ExactParams, FamilyCapability, Field, FieldDataType, GroupingStrategy, - HydraKind, HydraParams, MergeInputRule, MergedError, SamplingKind, SamplingParams, Schema, - SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, StatModelKind, StatModelParams, - StatisticKind, SummaryUpdate, WaveletKind, WaveletParams, -}; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ - apply_materialization_timings, ASAPOp, ExprSemantics, MaterializationAssignment, - NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, + ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, +}; +use asap_types::summary_type::{ + ExactKind, ExactParams, MergeInputRule, MergedError, SamplingKind, SamplingParams, + SketchAlgorithm, SketchStatistic, StatModelKind, StatModelParams, StatisticKind, + SummaryTypeCapability, SummaryUpdate, WaveletKind, WaveletParams, }; -use asap_types::physical::{Materialization, Medium}; use std::rc::Rc; /// KLL over `value` for one `region`, so states of different regions are /// disjoint. -fn state(k: u32, region: &str) -> Rc { - family_state( - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - Default::default(), - ), - region, - ) +fn state(_k: u32, region: &str) -> Rc { + family_state(FieldDataType::Sketch(SketchAlgorithm::Kll), region) } /// A `family` state over `value` for one `region`. fn family_state(family: FieldDataType, region: &str) -> Rc { @@ -57,10 +42,9 @@ fn family_state(family: FieldDataType, region: &str) -> Rc { .unwrap(); OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: only_region, - family, + summary_type: family, input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap() @@ -70,25 +54,21 @@ fn family_state(family: FieldDataType, region: &str) -> Rc { fn compatible_tumbling_windows_merge_structurally() { let root = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![state(200, "us"), state(200, "eu")], - newest_tumbling_windows: None, group_by: None, })) .unwrap(); root.validate_structure().unwrap(); assert_eq!(root.schema.fields.len(), 1); } -/// An empty merge, raw rows and differently sized state cannot masquerade as compatible tumbling windows. +/// An empty merge and raw rows cannot masquerade as compatible tumbling +/// windows. (State sizes are no longer part of the state's type: θ is a +/// candidate's `LogicalProperties::parameters`.) #[test] fn incompatible_merge_inputs_fail() { - for children in [ - vec![], - vec![state(200, "us"), state(300, "eu")], - vec![state(200, "us").children()[0].clone()], - ] { + for children in [vec![], vec![state(200, "us").children()[0].clone()]] { assert!( OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children, - newest_tumbling_windows: None, group_by: None, })) .is_err() @@ -96,72 +76,37 @@ fn incompatible_merge_inputs_fail() { } } -fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) -} -fn hydra( - algorithm: SketchAlgorithm, - params: SketchParams, - kind: HydraKind, - hydra_params: HydraParams, -) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::SharedMultiSubpopulation { - kind, - params: hydra_params, - }, - ) +fn sketch(algorithm: SketchAlgorithm) -> FieldDataType { + FieldDataType::Sketch(algorithm) } fn exact(kind: ExactKind, params: ExactParams) -> FieldDataType { FieldDataType::ExactAggregate(kind, params) } fn kll() -> FieldDataType { - sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ) + sketch(SketchAlgorithm::Kll) } fn hll() -> FieldDataType { - sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, - ) + sketch(SketchAlgorithm::Hll) } fn cms_with_heap() -> FieldDataType { - sketch( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 4, - heap_size: 10, - }, - ) + sketch(SketchAlgorithm::CmsWithHeap) } fn merge(children: Vec>) -> Result, String> { OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children, - newest_tumbling_windows: None, group_by: None, })) .map_err(|e| e.to_string()) } -/// The family-capability table, row by row: how each family's merge +/// The summary-type capability table, row by row: how each type's merge /// inputs must relate, and which statistics one state answers with the /// error of each from a merged state. Every statistic kind is probed /// through `answers` and `merged_error`, so the table and both lookups are /// pinned together. #[test] -fn family_capability_table() { +fn summary_type_capability_table() { use MergeInputRule::{Disjoint, DisjointInTime, MayOverlap}; - let (width, depth, heap_size) = (64, 4, 10); // (family, how its merge inputs must relate, its readouts). type Row = ( FieldDataType, @@ -237,101 +182,30 @@ fn family_capability_table() { }; let sketches = [ (kll(), SketchAlgorithm::Kll), - ( - sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - SketchAlgorithm::DDSketch, - ), + (sketch(SketchAlgorithm::DDSketch), SketchAlgorithm::DDSketch), (hll(), SketchAlgorithm::Hll), + (sketch(SketchAlgorithm::Cms), SketchAlgorithm::Cms), ( - sketch(SketchAlgorithm::Cms, SketchParams::Cms { width, depth }), - SketchAlgorithm::Cms, - ), - ( - sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width, depth }, - ), + sketch(SketchAlgorithm::CountSketch), SketchAlgorithm::CountSketch, ), - ( - sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size, - sketch_rows: depth, - sketch_cols: width, - layers: 8, - distinct_failure_probability: None, - }, - ), - SketchAlgorithm::UnivMon, - ), - ( - sketch( - SketchAlgorithm::Kmv, - SketchParams::Kmv { - k: 1024, - relative_error: None, - }, - ), - SketchAlgorithm::Kmv, - ), + (sketch(SketchAlgorithm::UnivMon), SketchAlgorithm::UnivMon), + (sketch(SketchAlgorithm::Kmv), SketchAlgorithm::Kmv), (cms_with_heap(), SketchAlgorithm::CmsWithHeap), ( - sketch( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - }, - ), + sketch(SketchAlgorithm::CountSketchWithHeap), SketchAlgorithm::CountSketchWithHeap, ), ( - hydra( - SketchAlgorithm::Cms, - SketchParams::Cms { width, depth }, - HydraKind::HydraCms, - HydraParams::HydraCms { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - ), + FieldDataType::Sketch(SketchAlgorithm::HydraCms), SketchAlgorithm::HydraCms, ), ( - hydra( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width, depth }, - HydraKind::HydraCountSketch, - HydraParams::HydraCountSketch { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - ), + FieldDataType::Sketch(SketchAlgorithm::HydraCountSketch), SketchAlgorithm::HydraCountSketch, ), ( - hydra( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - HydraKind::HydraKll, - HydraParams::HydraKll { - k: 200, - shared_buckets: width, - }, - ), + FieldDataType::Sketch(SketchAlgorithm::HydraKll), SketchAlgorithm::HydraKll, ), ]; @@ -358,7 +232,7 @@ fn family_capability_table() { SketchStatistic::TopK { k: 5 }, ]; for (family, merge_inputs, readouts) in rows { - let capability = FamilyCapability::of(&family); + let capability = SummaryTypeCapability::of(&family); assert_eq!(capability.merge_inputs, merge_inputs, "{family:?}"); assert_eq!(capability.readouts().to_vec(), readouts, "{family:?}"); for statistic in &statistics { @@ -396,16 +270,7 @@ fn merged_error_depends_on_the_readout() { key: ColumnRef::Named("k".into()), value: None, }; - let univmon = FamilyCapability::of(&sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 10, - sketch_rows: 4, - sketch_cols: 64, - layers: 8, - distinct_failure_probability: None, - }, - )); + let univmon = SummaryTypeCapability::of(&sketch(SketchAlgorithm::UnivMon)); for (readout, error) in [ (FrequencyL2, MergedError::AsIfBuiltOverUnion), (point(), MergedError::Exact), @@ -415,7 +280,7 @@ fn merged_error_depends_on_the_readout() { assert_eq!(univmon.merged_error(&readout), Some(error), "{readout:?}"); } assert_eq!(univmon.merged_error(&TopK { k: 5 }), None); - let heap = FamilyCapability::of(&cms_with_heap()); + let heap = SummaryTypeCapability::of(&cms_with_heap()); assert_eq!( heap.merged_error(&TopK { k: 5 }), Some(MergedError::Approximate) @@ -423,13 +288,7 @@ fn merged_error_depends_on_the_readout() { assert_eq!(heap.merged_error(&point()), None); // Count Sketch's signed counters give no total: no merged error for a // bare point count, and one build's guarantee for an item's frequency. - let count_sketch = FamilyCapability::of(&sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: 64, - depth: 5, - }, - )); + let count_sketch = SummaryTypeCapability::of(&sketch(SketchAlgorithm::CountSketch)); assert!(!count_sketch.answers(&point())); assert_eq!(count_sketch.merged_error(&point()), None); let item = SketchStatistic::PointCount { @@ -442,17 +301,20 @@ fn merged_error_depends_on_the_readout() { ); // An exact accumulator is read as its finalized value, not a readout. let sum = exact(ExactKind::Sum, ExactParams::Sum); - assert_eq!(FamilyCapability::of(&sum).merged_error(&point()), None); + assert_eq!(SummaryTypeCapability::of(&sum).merged_error(&point()), None); assert_eq!( - FamilyCapability::of(&hll()).merged_error(&Cardinality), + SummaryTypeCapability::of(&hll()).merged_error(&Cardinality), Some(MergedError::AsIfBuiltOverUnion) ); assert_eq!( - FamilyCapability::of(&kll()).merged_error(&SketchStatistic::Quantile { q: 0.99 }), + SummaryTypeCapability::of(&kll()).merged_error(&SketchStatistic::Quantile { q: 0.99 }), Some(MergedError::AsIfBuiltOverUnion) ); let irate = exact(ExactKind::IRate, ExactParams::IRate); - assert_eq!(FamilyCapability::of(&irate).merged_error(&point()), None); + assert_eq!( + SummaryTypeCapability::of(&irate).merged_error(&point()), + None + ); } /// A structurally validated `SummaryEstimate` of `query` over a `family` @@ -472,17 +334,8 @@ fn estimate(family: FieldDataType, query: SketchStatistic) -> Result bool { use SketchAlgorithm as A; use SketchStatistic as Q; @@ -517,6 +370,8 @@ fn accepted_by_category(algorithm: &SketchAlgorithm, query: &SketchStatistic) -> A::Cms | A::CountSketch => matches!(query, Q::PointCount { .. }), A::CmsWithHeap | A::CountSketchWithHeap => matches!(query, Q::TopK { .. }), A::UnivMon => !matches!(query, Q::Quantile { .. }), + A::HydraKll => matches!(query, Q::Quantile { .. }), + A::HydraCms | A::HydraCountSketch => matches!(query, Q::PointCount { .. }), _ => unreachable!("{algorithm:?} is not probed"), } } @@ -529,93 +384,25 @@ fn accepted_by_category(algorithm: &SketchAlgorithm, query: &SketchStatistic) -> /// answer. #[test] fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { - let (width, depth) = (64, 4); + use SketchAlgorithm as A; let mut families: Vec = [ - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - SketchParams::DDSketch { alpha: 0.01 }, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, - SketchParams::Kmv { - k: 1024, - relative_error: None, - }, - SketchParams::Cms { width, depth }, - SketchParams::CountSketch { width, depth }, - SketchParams::CmsWithHeap { - width, - depth, - heap_size: 10, - }, - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size: 10, - }, - SketchParams::UnivMon { - heap_size: 10, - sketch_rows: depth, - sketch_cols: width, - layers: 8, - distinct_failure_probability: None, - }, + A::Kll, + A::DDSketch, + A::Hll, + A::Kmv, + A::Cms, + A::CountSketch, + A::CmsWithHeap, + A::CountSketchWithHeap, + A::UnivMon, ] .into_iter() - .map(|params| { - use SketchAlgorithm as A; - let algorithm = match ¶ms { - SketchParams::Kll { .. } => A::Kll, - SketchParams::DDSketch { .. } => A::DDSketch, - SketchParams::Hll { .. } => A::Hll, - SketchParams::Kmv { .. } => A::Kmv, - SketchParams::Cms { .. } => A::Cms, - SketchParams::CountSketch { .. } => A::CountSketch, - SketchParams::CmsWithHeap { .. } => A::CmsWithHeap, - SketchParams::CountSketchWithHeap { .. } => A::CountSketchWithHeap, - SketchParams::UnivMon { .. } => A::UnivMon, - }; - sketch(algorithm, params) - }) + .map(sketch) .collect(); families.extend([ - hydra( - SketchAlgorithm::Cms, - SketchParams::Cms { width, depth }, - HydraKind::HydraCms, - HydraParams::HydraCms { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - ), - hydra( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width, depth }, - HydraKind::HydraCountSketch, - HydraParams::HydraCountSketch { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - ), - hydra( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - HydraKind::HydraKll, - HydraParams::HydraKll { - k: 200, - shared_buckets: width, - }, - ), + FieldDataType::Sketch(SketchAlgorithm::HydraCms), + FieldDataType::Sketch(SketchAlgorithm::HydraCountSketch), + FieldDataType::Sketch(SketchAlgorithm::HydraKll), ]); let statistics = [ SketchStatistic::PointCount { @@ -633,22 +420,24 @@ fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { SketchStatistic::TopK { k: 5 }, ]; for family in families { - let FieldDataType::Sketch(kind, _) = &family else { + let FieldDataType::Sketch(kind) = &family else { unreachable!() }; for query in &statistics { let fixed = matches!( - (kind.algorithm(), query.kind()), + (kind, query.kind()), ( SketchAlgorithm::UnivMon, StatisticKind::ItemFrequency | StatisticKind::TopK - ) | (SketchAlgorithm::CountSketch, StatisticKind::Total) - | ( - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, - StatisticKind::ItemFrequency, - ) + ) | ( + SketchAlgorithm::CountSketch | SketchAlgorithm::HydraCountSketch, + StatisticKind::Total + ) | ( + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, + StatisticKind::ItemFrequency, + ) ); - let before = accepted_by_category(kind.algorithm(), query); + let before = accepted_by_category(kind, query); let after = estimate(family.clone(), query.clone()).is_ok(); assert_eq!(after, before != fixed, "{family:?} {query:?}"); } @@ -667,80 +456,5 @@ fn summary_merge_is_permissive_about_the_family() { ]) .unwrap(); merged.validate_structure().unwrap(); - assert_eq!(FamilyCapability::of(&irate()).merge_inputs, None); -} - -/// Idempotent families (HLL, exact Min/Max) merge states that may share -/// rows, and the union of identical selections is that selection; counting -/// families (exact Sum, KLL) still need disjoint selections. -#[test] -fn idempotent_families_merge_overlapping_states() { - let same_region = |family: FieldDataType| { - merge(vec![ - family_state(family.clone(), "us"), - family_state(family, "us"), - ]) - }; - for family in [hll(), exact(ExactKind::Max, ExactParams::Max)] { - let merged = same_region(family.clone()).unwrap_or_else(|e| panic!("{family:?}: {e}")); - merged.validate_structure().unwrap(); - assert_eq!( - merged.coverage().unwrap().selection(), - family_state(family, "us").coverage().unwrap().selection() - ); - } - for family in [exact(ExactKind::Sum, ExactParams::Sum), kll()] { - let error = same_region(family).unwrap_err(); - assert!(error.contains("not proven disjoint"), "{error}"); - } -} - -/// `SummaryMerge` takes the timing of its consumer, and reads tumbling-window states -/// stored at ingestion time (#509 Example 3/4 Pattern B1) or built at query -/// time (B2). Merging at ingestion time needs every tumbling window at ingestion time. -#[test] -fn summary_merge_timing() { - let tumbling_windows = [state(200, "us"), state(200, "eu")]; - let root = merge(tumbling_windows.to_vec()).unwrap(); - let timings = |assignment: &MaterializationAssignment| { - let timed = - apply_materialization_timings(&root, assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let children = timed.children(); - ( - timed.timing.unwrap(), - children - .iter() - .map(|c| c.timing.unwrap()) - .collect::>(), - ) - }; - use ExecutionTiming::{IngestionTime, QueryTime}; - assert_eq!( - timings(&MaterializationAssignment::default()), - (QueryTime, vec![QueryTime, QueryTime]) - ); - let at_ingestion = Materialization::MatAtIngestion(Medium::Memory); - let mut stored_tumbling_windows = MaterializationAssignment::default(); - for tumbling_window in &tumbling_windows { - stored_tumbling_windows.set(tumbling_window, at_ingestion); - } - assert_eq!( - timings(&stored_tumbling_windows), - (QueryTime, vec![IngestionTime, IngestionTime]) - ); - // A `SummaryAgg` without an assignment runs at query time, so a merge - // stored at ingestion time needs its tumbling windows there too. - let mut only_merge = MaterializationAssignment::default(); - only_merge.set(&root, at_ingestion); - assert!( - apply_materialization_timings(&root, &only_merge, &mut NodeToTimedNodeCache::new()) - .is_err() - ); - let mut stored_merge = MaterializationAssignment::with_default(at_ingestion); - stored_merge.set(&root, at_ingestion); - assert_eq!( - timings(&stored_merge), - (IngestionTime, vec![IngestionTime, IngestionTime]) - ); + assert_eq!(SummaryTypeCapability::of(&irate()).merge_inputs, None); } diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index b37d77385..5dec0c4a7 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -1,21 +1,13 @@ -//! A `SummaryTumblingWindowChain` of tumbling windows stands for its tumbling windows: it covers what one -//! build over its whole window does, and expands into the `SummaryAgg` -//! tumbling windows the executor runs (#509 Pass 2, window-composition rule). -use std::ops::Bound; +//! A `SummaryTumblingWindowChain`: one state per window of its width, with +//! no range and no window count (concepts doc, section 3, "Chain groups"). use std::rc::Rc; use std::time::Duration; -use asap_types::ir::operator::operator_properties::{Reduction, Source, TimeShift}; -use asap_types::ir::operator::tumbling_window_chain::{ - expand_tumbling_window_chains, merged_chain, -}; +use asap_types::ir::operator::arguments::{Reduction, Source}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, -}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; -use asap_types::test_support::{tumbling_window, whole_window_build}; +use asap_types::summary_type::{SketchAlgorithm, SummaryUpdate}; const MINUTE: u64 = 60_000; @@ -35,190 +27,25 @@ fn scan() -> Rc { .unwrap() } -fn shifted(child: Rc, offset_ms: i64) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms, - at: None, - }, - child, - })) - .unwrap() -} - fn kll() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::Kll) } -/// Five 1-min KLL tumbling windows per series over `input`. +/// 1-min KLL tumbling windows per series over `input`. fn chain(input: Rc) -> Rc { OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child: input, - family: kll(), + summary_type: kll(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), filter: None, tumbling_window_width_ms: MINUTE, - tumbling_window_count: 5, })) .unwrap() } -fn merge(children: Vec>) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap() -} - -fn relative_time(node: &OperatorNode) -> (Bound, Bound) { - let coverage = node.coverage().expect("coverage"); - let [selection] = coverage.selection() else { - panic!("one box: {coverage:?}") - }; - selection.relative_time().expect("a time window") -} - -/// The merged chain is a state of the build over the whole 5-min window: -/// the same schema and coverage `(-5m, 0]`, and it validates. -#[test] -fn merged_tumbling_windows_cover_the_whole_window() { - let chain = chain(scan()); - let merged = merge(vec![Rc::clone(&chain)]); - merged.validate_structure().unwrap(); - assert!(merged_chain(&merged).is_some_and(|c| std::ptr::eq(c, &*chain))); - let whole = whole_window_build(&chain).unwrap(); - let Some(NonASAPOp::TimeRange { range, kind, .. }) = whole.children()[0].non_asap() else { - panic!("{:?}", whole.operator) - }; - assert_eq!( - (*range, *kind), - (Duration::from_millis(5 * MINUTE), TimeRangeKind::Range) - ); - assert_eq!(merged.schema, whole.schema); - assert_eq!(merged.coverage(), whole.coverage()); - assert_eq!( - relative_time(&merged), - (Bound::Excluded(-5 * MINUTE as i64), Bound::Included(0)) - ); -} - -/// Tumbling window `i` is the build over `(-(i+1)·1m, -i·1m]`, the query's own offset -/// folded into one shift; the tumbling windows are disjoint and tile the window. -#[test] -fn each_tumbling_window_is_one_shifted_minute() { - let offset = 3_600_000; - let chain = chain(shifted(scan(), offset)); - for i in 0..5 { - let tumbling_window = tumbling_window(&chain, i).unwrap(); - let back = offset + i64::from(i) * MINUTE as i64; - assert_eq!( - relative_time(&tumbling_window), - ( - Bound::Excluded(-(back + MINUTE as i64)), - Bound::Included(-back) - ), - "tumbling window {i}" - ); - let range = tumbling_window.children()[0]; - let Some(NonASAPOp::TimeShift { shift, child }) = range.children()[0].non_asap() else { - panic!("tumbling window {i} reads a shift") - }; - assert_eq!(shift.offset_ms, back); - assert!(matches!(child.non_asap(), Some(NonASAPOp::Scan { .. }))); - } -} - -/// Expansion gives the executor a merge over the five tumbling windows, newest first, -/// which covers what the merged chain does; a chain read without a merge -/// does not expand. -#[test] -fn expansion_merges_the_tumbling_windows() { - let chain = chain(scan()); - let merged = merge(vec![Rc::clone(&chain)]); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: Rc::clone(&merged), - query: SketchStatistic::Quantile { q: 0.99 }, - })) - .unwrap(); - let expanded = expand_tumbling_window_chains(&estimate).unwrap(); - expanded.validate_structure().unwrap(); - assert_eq!(expanded.schema, estimate.schema); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = expanded.asap() else { - panic!("{:?}", expanded.operator) - }; - let Some(ASAPOp::SummaryMerge { children, .. }) = summary_input.asap() else { - panic!("{:?}", summary_input.operator) - }; - assert_eq!(children.len(), 5); - for (i, child) in children.iter().enumerate() { - assert_eq!( - **child, - tumbling_window(&chain, i as u32).unwrap(), - "tumbling window {i}" - ); - } - assert_eq!(summary_input.coverage(), merged.coverage()); - assert!(expand_tumbling_window_chains(&chain).is_err()); -} - -/// Expanding a timed chain carries its timing and materialization onto its -/// tumbling windows: a chain materialized at ingestion time expands into tumbling windows each -/// built and stored at ingestion time, over ranges and shifts that run at -/// ingestion time too, while the merge reading them stays at query time. -/// An unmaterialized chain expands into query-time tumbling windows. -#[test] -fn expansion_carries_the_chains_timing_onto_its_tumbling_windows() { - use asap_types::ir::properties::ExecutionTiming; - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - use asap_types::physical::{Materialization, Medium}; - let at_ingestion = Materialization::MatAtIngestion(Medium::Memory); - for (materialization, timing) in [ - (at_ingestion, ExecutionTiming::IngestionTime), - (Materialization::NotMaterialized, ExecutionTiming::QueryTime), - ] { - let chain = chain(shifted(scan(), MINUTE as i64)); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: merge(vec![Rc::clone(&chain)]), - query: SketchStatistic::Quantile { q: 0.99 }, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&chain, materialization); - let timed = - apply_materialization_timings(&estimate, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let expanded = expand_tumbling_window_chains(&timed).unwrap(); - let merged = expanded.children()[0]; - assert_eq!(merged.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(merged.children().len(), 5); - for tumbling_window in merged.children() { - assert_eq!(tumbling_window.timing, Some(timing), "{materialization:?}"); - assert_eq!(tumbling_window.materialization, materialization); - for below in OperatorNode::reachable(tumbling_window).iter().skip(1) { - assert_eq!(below.timing, Some(timing), "{}", below.operator.kind_name()); - assert_eq!(below.materialization, Materialization::NotMaterialized); - } - } - } -} - -/// A chain needs a positive width and count, and reads the instant vector -/// a range selector would. +/// A chain needs a positive width, and reads the instant vector a range +/// selector would. #[test] fn a_chain_reads_an_instant_vector() { let ranged = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { @@ -233,265 +60,28 @@ fn a_chain_reads_an_instant_vector() { }; let empty = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child: scan(), - family: kll(), + summary_type: kll(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), filter: None, - tumbling_window_width_ms: MINUTE, - tumbling_window_count: 0, + tumbling_window_width_ms: 0, })) .unwrap(); assert!(empty.validate_structure().is_err()); } -/// Two merges over one chain (two queries reading the same tumbling windows) expand -/// into one set of tumbling windows: the same five `SummaryAgg` nodes, not ten. -#[test] -fn merges_over_one_chain_share_its_tumbling_windows() { - use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; - use asap_types::ir::QueryRoot; - let shared = chain(scan()); - let roots = [ - QueryRoot::Operator(merge(vec![Rc::clone(&shared)])), - QueryRoot::Operator(merge(vec![shared])), - ]; - let expansion = expand_tumbling_window_chains_in_workload(&roots).unwrap(); - assert_eq!(expansion.chains.len(), 1); - assert_eq!(expansion.chains[0].tumbling_windows.len(), 5); - let expanded = expansion.roots; - let tumbling_windows = |root: &QueryRoot| match root { - QueryRoot::Operator(node) => node.children().into_iter().cloned().collect::>(), - QueryRoot::Scalar(_) => unreachable!(), - }; - let (first, second) = ( - tumbling_windows(&expanded[0]), - tumbling_windows(&expanded[1]), - ); - assert_eq!(first.len(), 5); - assert!(first.iter().zip(&second).all(|(a, b)| Rc::ptr_eq(a, b))); -} - -fn newest(chain: Rc, newest_tumbling_windows: u32) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![chain], - newest_tumbling_windows: Some(newest_tumbling_windows), - group_by: None, - })) - .unwrap() -} - -/// A merge of the newest 2 of 5 tumbling windows is a state of one build over the -/// newest 2 minutes: coverage `(-2m, 0]`, the window build's schema, and -/// a valid structure. -#[test] -fn a_merge_of_the_newest_tumbling_windows_covers_their_window() { - use asap_types::ir::operator::tumbling_window_chain::merged_window_build; - let partial = newest(chain(scan()), 2); - partial.validate_structure().unwrap(); - let build = merged_window_build(&partial).unwrap(); - let Some(NonASAPOp::TimeRange { range, .. }) = build.children()[0].non_asap() else { - panic!("{:?}", build.operator) - }; - assert_eq!(*range, Duration::from_millis(2 * MINUTE)); - assert_eq!(partial.schema, build.schema); - assert_eq!(partial.coverage(), build.coverage()); - assert_eq!( - relative_time(&partial), - (Bound::Excluded(-2 * MINUTE as i64), Bound::Included(0)) - ); -} - -/// `newest_tumbling_windows` must name between 1 and fewer than all tumbling windows of the one -/// chain the merge reads; anything else is rejected. -#[test] -fn a_merge_of_the_newest_tumbling_windows_reads_one_chain_and_fewer_than_all_tumbling_windows() { - let chain = chain(scan()); - let build = whole_window_build(&chain).unwrap(); - for (children, newest_tumbling_windows) in [ - (vec![Rc::clone(&chain)], 0), - (vec![Rc::clone(&chain)], 5), - (vec![Rc::clone(&chain)], 6), - (vec![Rc::clone(&chain), Rc::clone(&chain)], 2), - (vec![Rc::new(build)], 1), - ] { - let merge = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: Some(newest_tumbling_windows), - group_by: None, - })); - assert!(merge.is_err(), "newest {newest_tumbling_windows}"); - } -} - -/// A merge of the newest 2 tumbling windows and a merge of all 5 over one chain -/// expand into one set of tumbling windows: the partial merge reads the first 2 of -/// the full merge's 5 nodes, newest first, and the chain lists all 5. -#[test] -fn a_merge_of_the_newest_tumbling_windows_reads_the_first_tumbling_windows_of_the_chain() { - use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; - use asap_types::ir::QueryRoot; - let shared = chain(scan()); - let roots = [ - QueryRoot::Operator(newest(Rc::clone(&shared), 2)), - QueryRoot::Operator(merge(vec![shared])), - ]; - let expansion = expand_tumbling_window_chains_in_workload(&roots).unwrap(); - assert_eq!(expansion.chains.len(), 1); - assert_eq!(expansion.chains[0].tumbling_windows.len(), 5); - let tumbling_windows = |root: &QueryRoot| match root { - QueryRoot::Operator(node) => node.children().into_iter().cloned().collect::>(), - QueryRoot::Scalar(_) => unreachable!(), - }; - let (partial, full) = ( - tumbling_windows(&expansion.roots[0]), - tumbling_windows(&expansion.roots[1]), - ); - assert_eq!((partial.len(), full.len()), (2, 5)); - assert!(partial.iter().zip(&full).all(|(a, b)| Rc::ptr_eq(a, b))); - for (root, window) in expansion.roots.iter().zip([2, 5]) { - let QueryRoot::Operator(node) = root else { - unreachable!() - }; - assert_eq!( - relative_time(node), - (Bound::Excluded(-window * MINUTE as i64), Bound::Included(0)) - ); - } -} - -/// `newest_tumbling_windows` is left out of the JSON when `None`, and JSON without it -/// reads as `None`; `Some(n)` round-trips. +/// JSON written before #1163 names the chain `PaneChain` and its width +/// `pane_width_ms`; it reads as today's operator. #[test] -fn newest_tumbling_windows_serializes_only_when_set() { - let full: ASAPOp<()> = ASAPOp::SummaryMerge { - children: vec![()], - newest_tumbling_windows: None, - group_by: None, - }; - let json = serde_json::to_string(&full).unwrap(); - assert!(!json.contains("newest_tumbling_windows"), "{json}"); - let read: ASAPOp<()> = serde_json::from_str(r#"{"SummaryMerge":{"children":[null]}}"#).unwrap(); - assert_eq!(read, full); - let partial: ASAPOp<()> = ASAPOp::SummaryMerge { - children: vec![()], - newest_tumbling_windows: Some(6), - group_by: None, - }; - let json = serde_json::to_string(&partial).unwrap(); - assert_eq!(serde_json::from_str::>(&json).unwrap(), partial); -} - -/// A rollup of a chain's tumbling windows to no keys (every series' tumbling windows into one -/// state) is no tumbling-window merge, and expansion keeps its keys: the expanded -/// rollup covers what the planned one does, one build over the whole -/// window and every series. -#[test] -fn expansion_keeps_a_rollup_of_the_tumbling_windows() { - let chain = chain(scan()); - let rollup = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&chain)], - group_by: Some(vec![]), - newest_tumbling_windows: None, - })) - .unwrap(); - assert!(merged_chain(&rollup).is_none()); - let expanded = expand_tumbling_window_chains(&rollup).unwrap(); - let Some(ASAPOp::SummaryMerge { - children, group_by, .. - }) = expanded.asap() - else { - panic!("{:?}", expanded.operator) - }; - assert_eq!((children.len(), group_by), (5, &Some(vec![]))); - let coverage = rollup.coverage().unwrap(); - assert_eq!(expanded.coverage(), Some(coverage.clone())); - let Some(ASAPOp::SummaryAgg { reduction, .. }) = coverage.definition().asap() else { - panic!("{:?}", coverage.definition().operator) - }; - assert_eq!(*reduction, Reduction::by(vec![])); - // A rollup merges every input state: it reads a merge of the newest - // tumbling windows, it is not one. - assert!(OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![chain], - newest_tumbling_windows: Some(2), - group_by: Some(vec![]), - })) - .is_err()); -} - -/// A rollup reads a merge of the newest tumbling windows: every series' newest 2 of 5 -/// tumbling windows into one state covers one build by no keys over the newest 2 -/// minutes. -#[test] -fn a_rollup_of_the_newest_tumbling_windows_covers_their_window() { - let rollup = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![newest(chain(scan()), 2)], - newest_tumbling_windows: None, - group_by: Some(vec![]), - })) - .unwrap(); - rollup.validate_structure().unwrap(); - let coverage = rollup.coverage().unwrap(); - let Some(ASAPOp::SummaryAgg { reduction, .. }) = coverage.definition().asap() else { - panic!("{:?}", coverage.definition().operator) - }; - assert_eq!(*reduction, Reduction::by(vec![])); +fn names_before_the_tumbling_window_rename_still_read() { + let operator = chain(scan()).operator.map_children(|_| ()); + let old = serde_json::to_string(&operator) + .unwrap() + .replace("SummaryTumblingWindowChain", "PaneChain") + .replace("tumbling_window_width_ms", "pane_width_ms"); + assert!(!old.contains("umbling"), "{old}"); assert_eq!( - coverage.selection()[0].relative_time(), - Some((Bound::Excluded(-2 * MINUTE as i64), Bound::Included(0))) + serde_json::from_str::>(&old).unwrap(), + operator ); } - -/// Expansion keeps every tumbling window and the identity of a chain two -/// readers share. -#[test] -fn expansion_preserves_every_tumbling_window_and_shared_reader() { - use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; - use asap_types::ir::QueryRoot; - let shared = chain(scan()); - let roots = [ - QueryRoot::Operator(merge(vec![shared.clone()])), - QueryRoot::Operator(merge(vec![shared])), - ]; - let result = expand_tumbling_window_chains_in_workload(&roots).unwrap(); - assert_eq!(result.chains.len(), 1); - assert_eq!(result.chains[0].tumbling_windows.len(), 5); - let QueryRoot::Operator(first) = &result.roots[0] else { - unreachable!() - }; - let QueryRoot::Operator(second) = &result.roots[1] else { - unreachable!() - }; - assert!(first - .children() - .iter() - .zip(second.children()) - .all(|(a, b)| Rc::ptr_eq(a, b))); -} - -/// JSON written before #1163 names the chain `PaneChain`, its fields -/// `tumbling_window_width_ms` and `tumbling_window_count`, and a merge's `newest_tumbling_windows`; it reads -/// as today's operators. -#[test] -fn names_before_the_tumbling_window_rename_still_read() { - let chain = chain(scan()).operator.map_children(|_| ()); - let merge: Operator<()> = Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![()], - newest_tumbling_windows: Some(3), - group_by: None, - }); - for operator in [chain, merge] { - let old = serde_json::to_string(&operator) - .unwrap() - .replace("SummaryTumblingWindowChain", "PaneChain") - .replace("tumbling_window_width_ms", "pane_width_ms") - .replace("tumbling_window_count", "pane_count") - .replace("newest_tumbling_windows", "newest_panes"); - assert!(!old.contains("umbling"), "{old}"); - assert_eq!( - serde_json::from_str::>(&old).unwrap(), - operator - ); - } -} diff --git a/docs/design_docs/architecture/evidence-dependent-candidates.md b/docs/design_docs/architecture/evidence-dependent-candidates.md index 98823f488..74bd32816 100644 --- a/docs/design_docs/architecture/evidence-dependent-candidates.md +++ b/docs/design_docs/architecture/evidence-dependent-candidates.md @@ -19,8 +19,8 @@ not turn an unknown guarantee into a satisfied accuracy requirement. ## Candidate state Here, **certified for a target** means Planner has a fully known -`AccuracyGuarantee` for the candidate and `AccuracyModel::satisfies(guarantee, -target)` returns `true`. **Uncertified** means Planner cannot make that claim: +`AccuracyGuarantee` for the candidate whose evaluated error is within the +target's (`asap_accuracy::target_limit`). **Uncertified** means Planner cannot make that claim: the guarantee is absent, contains unknown terms, or is known not to meet the target. An uncertified summary may still be a well-formed logical candidate; this label says nothing about whether the backend can physically execute it. @@ -73,7 +73,7 @@ not emit a `RejectedCandidate` for that case. The backend must still check summary family, window, and state-operation capabilities before deployment. -- Accuracy/domain: `AccuracyEvidenceProvider` supplies quantile domains and +- Accuracy/domain: `AccuracyEvidence` supplies measured guarantees; the data workload supplies quantile domains and propagation statistics. DDSketch ratio domains, Hydra shared-grid bounds, TopK margin intervals, HLL confidence, and composition row counts may be missing. A missing non-negative-value proof for relative-error composition @@ -134,7 +134,7 @@ All three start from PromQL text and exercise frontend lowering and planning. None runs a deployed query. For a workload containing `quantile_over_time(0.9,data[5m]) / -quantile_over_time(0.5,data[5m])`, search with `NoAccuracyEvidence` and a root +quantile_over_time(0.5,data[5m])`, search with no accuracy evidence and a root target may expose a direct DDSketch ratio candidate. Its missing domain proof is visible through `has_missing_accuracy_evidence()`. The backend examines the operand windows and its own data-domain facts, checks whether the denominator diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index 400b73675..b24531056 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -18,7 +18,7 @@ selected. | `PlanningWorkload.query_workload` | Query language and one-time/repeating query workloads | Yes | | `PlanningWorkload.data_workload` | How the data arrives, and facts about each source | No implicit default. Set `None` when unavailable; a PromQL query with a bare instant selector needs a nonzero `sample_interval` on the metric it reads. | | Frontend-specific dependencies (outside `PlanningWorkload`) | `SqlCatalog` for SQL | `SqlCatalog` is required for SQL lowering | -| `PlanningModels` | Accuracy model and evidence, `CostInputs` (cost model 1, 2 or 3, resource speed, measured operation costs), `DeploymentCapabilities` | `PlanningModels::builtin()` uses `BuiltinAccuracyModel`, no evidence, cost model 1 with illustrative unit costs, and a deployment that supports everything | +| `PlanningModels` | Accuracy evidence (`AccuracyEvidence`), `CostInputs` (cost model 1, 2 or 3, resource speed, measured operation costs), `DeploymentCapabilities` | `PlanningModels::builtin()` uses no evidence, cost model 1 with illustrative unit costs, and a deployment that supports everything | Frontend lowering (Stage 0) and Stages 1–3 run inside the workflow; they are not additional inputs. See [Inputs](#inputs) for the nested workload fields @@ -28,7 +28,7 @@ and [frontend dependencies](#frontend-specific-dependencies). | Output | Fields or contents | Meaning | |---|---|---| -| `PlanOutput` | `plans`: one `QueryPlan { entry_index, root }` per query; `selection`: Stage 3's `Selection` (`selected_dag`, `nodes_with_proven_non_negative_weights`, `cost`, `accepted_accuracy_per_query`, `pruned_candidates`, `search_report`), set by the default pass `Stage1ToStage3Pass`; `tumbling_window_retention_ms`: how long each kept tumbling-window chain keeps its tumbling windows; standalone scalar queries in `scalar_roots` and `scalar_plans` | The selected plan; `physical_asap_dag()` exports it as one `PhysicalASAPDAG` | +| `PlanOutput` | `plan`: the selected plan, one `PhysicalASAPDAG` (its roots in entry order, standalone scalar queries in `scalar_roots`) | The plan installed on the executor; Stage 3's `Selection` report is `run_planning`'s | | `PlanningRun` (from `asap_planner::run_planning`) | Every stage's output: Pass 1's per-computation candidates, the selected round's Pass 2 and Stage 2 outputs (with Stage 2's pruned candidates), the selection, the selected plan, the memo after each step and a report per tumbling-window-width round | For callers that inspect or display the alternatives | The selected plan is a planning artifact. ASAPPlanner does **not** deploy @@ -196,7 +196,7 @@ enum DataArrival { struct SourceFacts { series_count: Option, metric_type: Option, - values_non_negative: bool, + column_signs: BTreeMap, sample_interval: Option, stream_norms: Option, max_distinct_per_evaluation: Option, @@ -215,8 +215,8 @@ struct LabelSetDistinctValues { | `arrival` | Present; may be `Unknown` | Distinguishes data at rest, ingesting data, and mixed data. Ingestion-time builds are possible only for ingesting or mixed data. | | `rows_per_second` | With `Ingesting`/`Mixed` | Rows arriving per second over all sources, used to price scans and continuous maintenance. It must be finite and non-negative. A source with no `series_count` + `sample_interval` of its own is priced at its series share of it when every source declares `series_count`, else at all of it. | | `series_count` | Optional | Series in the source. Plan selection bounds a query's number of values `G`, and its input row count, by the series counts of the sources it reads (each branch of an `or` counted); unknown, they stay unbound. | -| `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `values_non_negative` and `sample_interval` apply to its series; counts, norms and bounds are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | -| `values_non_negative` | Optional; default `false` | Declares every sample non-negative, for a gauge as well as a counter. It is the sign a relative-error bound needs to carry through `sum` and `avg` (a per-series DDSketch quantile under `avg by`, say); without it, and for a gauge, that bound is unknown. | +| `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `column_signs` and `sample_interval` apply to its series; counts, norms and bounds are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | +| `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan`'s `column_sign` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. An undeclared column is `unknown`, except a PromQL counter's value. | | `sample_interval` | PromQL and MetricsQL; nonzero | How often each series of the metric is sampled. Bare instant selectors over it use it as their selection horizon; without it, such a query fails to lower. | | `stream_norms`, `max_distinct_per_evaluation` | Optional | Bounds that Count-Min/Count Sketch and HLL/UnivMon guarantees are bound with. | | `distinct_values_per_label` | Optional; default empty; each count nonzero | Distinct values each label (PromQL) or column (SQL) takes, e.g. `{"host": 1000, "job": 10}`. Plan selection prices a `by (k1, k2, ...)` aggregate or summary, and a partitioned sort or limit, at the product of its keys' counts, at most the `series_count` (series, or groups of rows) of the sources it reads and the rows it reads per evaluation. Over several sources a key's counts add. No count is assumed for a key some source read does not declare: the grouping is priced at the smaller of that `series_count` and those rows. Each count must be an upper bound, counting a missing label (or a NULL) as one more value: when every key of a grouped result declares one, their product (at most the series read) also bounds `G`, which sizes per-group sketches for a whole-result target; otherwise `G` keeps its series bound, or none for SQL. | @@ -248,15 +248,14 @@ They are explicit frontend function arguments, not one generic ### Planning evidence inputs **A model is planning logic; evidence is a scoped fact used by that logic.** -For example, `BuiltinAccuracyModel` knows how to combine error bounds, while -an operand-domain contract tells it whether a particular product or quotient -of absolute-error estimates has a bound. The former can be provided by a built-in default; the +For example, the accuracy rules know how to combine error bounds, while a +deployment's measurement (`AccuracyEvidence`) says what error a particular +summary configuration has. The former can be provided by a built-in default; the latter cannot be fabricated by one. | Input | Where it enters / default | Why it matters | |---|---|---| -| Accuracy model | `PlanningModels` (`AccuracyModel`; `BuiltinAccuracyModel` by default) | Composes candidate guarantees and checks them against requested accuracy. The model does not itself provide missing data-domain facts. | -| Accuracy/domain evidence | `PlanningModels` (`AccuracyEvidenceProvider`; `NoAccuracyEvidence` by default) | Input ranges, nonempty populations and similar facts can certify or rule out particular approximations. Missing facts remain unknown. | +| Accuracy evidence | `PlanningModels` (`AccuracyEvidence`; none by default) | Measured guarantees per summary configuration, read before the theoretical guarantee. Missing measurements leave the theoretical guarantee. | | Cost model | `CostInputs.cost_model` (cost model 1 by default) | Turns each valid candidate's resource usage into a choice. The illustrative unit costs are not a measured deployment cost. | | Measured cost evidence | `CostInputs.measured_operation_costs` | Measured CPU time per unit of each operator kind, at ingestion or query time, replaces the formulas for the nodes it prices (a sketch interpolated between measured sizes); the other nodes' formulas are scaled by one calibration factor fitted from the measured ones. | | Runtime capabilities | `DeploymentCapabilities` (`supported_summaries`, `raw_data_retained`, `query_time_results_retained`) | Prevents choosing a summary, raw-data scan or kept state the intended executor cannot provide. | @@ -279,21 +278,12 @@ candidate whose output has no guarantee for an approximate target. ### `PlanOutput` -`PlanOutput` holds the selected plan and Stage 3's record of the selection: - -* `plans`: one plan root per query, in `QueryWorkload::entries()` order; -* `selection.selected_dag`: the selected plan as a `DagOfSelectedCandidates`: - the physical candidate each node runs (one node per run, so a candidate - run twice is two nodes), the edge feeding each input of each node, and the - node producing each query output; -* `selection.cost`: the selected plan's resource usage and, under cost - models 1 and 2, its cost per second; -* `selection.accepted_accuracy_per_query`: per query, the accuracy metric - Stage 3 accepted; -* `selection.pruned_candidates`: the candidates Stage 3 ruled out; -* `selection.search_report`: how the search ended (`Optimal`: the MILP - runs to proven optimality, with no time limit), its lower bound, the - MILP's size and its times. +`PlanOutput` holds the selected plan, `plan`: one `PhysicalASAPDAG`, whose +`roots` answer the operator queries and `scalar_roots` the standalone scalar +queries, in `QueryWorkload::entries()` order. Its cost is +`asap_cost::plan_cost` of it. Stage 3's own report, `Selection` +(`accepted_accuracy_per_query`, `pruned_candidates` and the selection MILP's +size and times), is `run_planning`'s, beside the plan. Planner selects because it now has the inputs a backend used to own: the deployment declares its cost model, measurements and capabilities, and every @@ -305,7 +295,7 @@ alternative stays visible in `PlanningRun`. PlanningWorkload + frontend dependencies + PlanningModels -> Stage 0: frontend lowering, one root per normalized query entry -> Stage 1: Pass 1 per-computation candidates, Pass 2 sharing, tumbling windows, reuse - -> Stage 2: one physical candidate per materialization assignment + -> Stage 2: one physical candidate per physical properties assignment -> Stage 3: check, measure and select -> PlanOutput ``` diff --git a/docs/design_docs/architecture/physical-plan-integration.md b/docs/design_docs/architecture/physical-plan-integration.md index a9961b710..4d42aaab9 100644 --- a/docs/design_docs/architecture/physical-plan-integration.md +++ b/docs/design_docs/architecture/physical-plan-integration.md @@ -461,9 +461,9 @@ a numeric entity key is not a score. Exact accumulator inputs are explicitly finalized before row operators consume them. None of these operations proves candidate completeness; that evidence belongs to the semi-join's pruning step. -The logical DAG carries no execution layout: `apply_materialization_timings` -writes each node's timing and materialization from a -`MaterializationAssignment` before export (every sub-DAG `NotMaterialized` by -default, so all query time). The exported DAG carries both, and the deployment +The logical DAG carries no execution layout: `apply_physical_properties` +writes each node's timing and lifetime from a +`PhysicalPropertiesAssignment` before export (no sub-DAG materialized by default, so all +query time). The exported DAG carries both, and the deployment executes it as given; this is not a claim that every deployment has implemented every placement. diff --git a/docs/design_docs/architecture/planner-runtime-contract.md b/docs/design_docs/architecture/planner-runtime-contract.md index 69813bc5f..dbab5b3cc 100644 --- a/docs/design_docs/architecture/planner-runtime-contract.md +++ b/docs/design_docs/architecture/planner-runtime-contract.md @@ -14,7 +14,7 @@ execution remain downstream. The | Layer | Owner | Examples | |---|---|---| | Logical candidate semantics | ASAPPlanner | Query rewrite; summary family and parameters; grouping; accuracy guarantees when established. | -| Summary materialization and realization selection | ASAPPlanner Stage 2 materialization (#509) | `NotMaterialized`, `MatAtIngestion(medium)` or `MatAtQuery(medium)` per sub-DAG, and how long a stored output is kept; direct build or incremental update; window implementations compared using provider evidence. | +| Summary materialization and realization selection | ASAPPlanner Stage 2 materialization (#509) | a timing and a lifetime per sub-DAG (`None`, `OneEvaluation(medium)` or `AcrossEvaluations(medium)`), and how much a materialized output retains; direct build or incremental update; window implementations compared using provider evidence. | | Concrete implementation and deployment | ASAPQuery-backend and its workload optimizer | Library and data-structure implementation, exact tumbling-window layout, placement, sharding, storage, transmission, materialization IDs, executor configuration, and workload-wide assignment. | ASAPCollector and the ASAPQuery data plane execute the compiled downstream diff --git a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md index 3011064f3..dfc46531b 100644 --- a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md +++ b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md @@ -101,7 +101,7 @@ pub struct QueryPlan { ``` `Stage1ToStage3Pass` returns the selected plan's roots, as Stage 2's realizer -built them with each node's timing and materialization, and with the weight +built them with each node's timing and lifetime, and with the weight proofs Stage 3 made applied; `PlanOutput::physical_asap_dag()` exports them. Physical ASAP-aware optimization decides per sub-DAG whether to materialize, and whether at ingestion or query time, from #604 on. @@ -125,9 +125,9 @@ tracked in #580. | Step | Call | |---|---| | Prepare roots | Entries in `entries()` order, the accuracy target from each entry's `requirements`; PromQL operator roots carry the series identity (`with_promql_series_identity`) where it has a realization | -| Stage 1: logical ASAP-aware optimization | `generate_per_computation_logical_memo_candidates` (Pass 1), then `generate_logical_asap_memo_candidates` (Pass 2) over every tumbling-window width | +| Stage 1: logical ASAP-aware optimization | `pass1_memo` (Pass 1), then `generate_logical_asap_memo_candidates` (Pass 2) over every tumbling-window width | | Stage 2: physical ASAP-aware optimization | `generate_physical_asap_memo_candidates` | -| Stage 3: plan selection | `select_plan`: check accuracy and latency, price under the cost model, select one plan; the facade builds it with Stage 2's `AssignmentRealizer::realize` | +| Stage 3: plan selection | `select_plan`: select one plan under the cost model and build it (`PhysicalASAPDAG`) | `run_planning` runs these steps over the query-language frontends' roots (Stage 0) and returns `PlanningRun`: Pass 1's candidates, the Pass 2 and @@ -233,7 +233,7 @@ pass no longer runs these steps; another pass need not run any of them. | Crate | What it holds | |---|---| | `asap-types` | `ParsedWorkload` | -| `asap-plan-selection` | Stage 3: `select_plan`, `Selection`, `SearchReport`, `AccuracyCheckInputs` | +| `asap-plan-selection` | Stage 3: `select_plan`, `Selection` | | `asap-planner` *(new)* | `e2e_plan`, `UserInput`, `FrontendInput`, lowering dispatch; `OptimizationPass`, `OptimizationInput`, `PlanOutput`, `PlanningModels`, `optimize`, `PassRegistry`, `Stage1ToStage3Pass`, `run_planning` | ```text @@ -247,8 +247,8 @@ asap-planner ──┬──> asap-frontend-{sql, promql, metricsql} frontend; before it, the sole facade re-exporting more than one was `asap-devtools`, a developer-tools crate. It is also the only crate that depends on every stage, so it orchestrates them; `PlanningModels` lives in -it, and it gives each stage only its part (Stage 3 gets -`AccuracyCheckInputs` and `CostInputs`). +it, and it gives each stage only its part (Pass 2 gets the accuracy +evidence, Stage 3 `CostInputs`). --- diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 60aaa2d15..12004cb8b 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -201,8 +201,10 @@ cost model. ## Composing guarantees through a DAG -`AccuracyModel` supplies local guarantees, propagation rules and target -satisfaction. Its default implementation is conservative: an unregistered +The accuracy crate supplies theoretical guarantees (`theoretical_guarantee`), +propagation rules and the comparison limit of a target (`target_limit`); the +deployment's measurements (`AccuracyEvidence::measured_guarantee`) take precedence over a +theoretical guarantee. The rules are conservative: an unregistered composition over approximate inputs is rejected rather than treated as exact. The principal composition rules are: @@ -264,8 +266,7 @@ configuration or candidate is needed. ## Evidence and trust boundaries -`AccuracyEvidenceProvider` supplies estimator contracts, operand domains -and propagation evidence. The evidence must cover the population to which the +`AccuracyEvidence` supplies the deployment's measured guarantees. The evidence must cover the population to which the claimed guarantee applies: sources, filters, grouping, evaluation windows and all merged tumbling windows. Evidence for a narrower population cannot silently certify a wider one. Data-workload facts are declared: the planner has no clock, so the caller @@ -376,13 +377,15 @@ The `asap_accuracy` crate separates these responsibilities: ```text accuracy/src/ -├── lib.rs # Public interfaces and unified entry point (satisfies, per-input targets) -├── model.rs # AccuracyModel, AccuracyEvidenceProvider and their built-in defaults -├── evidence.rs # Evidence and source contracts +├── lib.rs # Re-exports: the public interfaces +├── evidence.rs # AccuracyEvidence: the deployment's measured accuracy +├── requirements.rs # producer_satisfies_logical_requirements (α^log), target_limit +├── error_propagation.rs # ErrorPropagation: a consumer's output error from its inputs' +├── propagation_stats.rs # Evidence and source contracts ├── composition.rs # Error propagation across computations ├── budget.rs # Splitting a query's accuracy budget across approximate inputs ├── function_rules.rs # Error composition of scalar functions over approximate inputs -└── estimators/ +└── guarantee/ ├── mod.rs # Family/evaluation dispatch and source-contract integration ├── kll.rs ├── ddsketch.rs @@ -408,10 +411,10 @@ Adding an estimator or composition requires: 5. Tests from target and evidence through sizing, composition and selection, including missing evidence and infeasible targets. Algorithm-specific mathematics belongs behind this common contract. It does -not require a new selection rule for every sketch. Deployment extensions to -`AccuracyModel` remain possible, but carry the same obligation to justify +not require a new selection rule for every sketch. A deployment's measured +guarantees (`AccuracyEvidence`) carry the same obligation to justify metrics, assumptions and propagation. For implementation details, see the [accuracy crate](../../../crates/accuracy/src/lib.rs), -[guarantee representation](../../../crates/types/src/ir/properties/guarantee.rs), and +[guarantee representation](../../../crates/types/src/accuracy/guarantee.rs), and [accuracy propagation companion](../../develop_docs/end-to-end-accuracy-guarantees.md). diff --git a/docs/design_docs/concepts/glossary.md b/docs/design_docs/concepts/glossary.md index c27400a31..8b52e6ef6 100644 --- a/docs/design_docs/concepts/glossary.md +++ b/docs/design_docs/concepts/glossary.md @@ -38,7 +38,7 @@ A concrete execution and deployment choice: runtime topology of data lifecycle s ## Materialized view -A node of the physical DAG whose output is kept across evaluations (`MatAtIngestion` or `MatAtQueryKeptAcrossEvaluations`), with how long it is kept (`retention_ms` on the node). A `MatAtQuery` node is not a view: its output is discarded after one evaluation. Code: `MaterializedView`, `PhysicalASAPDAG::materialized_views`. +A node of the physical DAG whose output is materialized across evaluations (lifetime `AcrossEvaluations`, at ingestion time or at query time), with how long it is materialized (`retention_ms` on the node). A node materialized for `OneEvaluation` is not a view: its output is discarded after one evaluation. Code: `MaterializedView`, `PhysicalASAPDAG::materialized_views`. ## Raw data sample diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index ea9d9e57b..6b7fe81d9 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -44,12 +44,12 @@ Implemented: population, including membership changes. - `EvaluatePopulation { child, evaluation }`: read an aggregate or TopK prefix from a maintained population. -- `SummaryMerge { children, newest_tumbling_windows, group_by }`: merge summary states with identical +- `SummaryMerge { children, group_by }`: merge summary states with identical schemas (same family, parameters and grouping) into the state of their union; its coverage check rejects inputs whose selections overlap. With `group_by` (a rollup) the output is keyed by those input group columns, and - each output state merges the input states of its key; `newest_tumbling_windows` - merges only the newest tumbling windows of one tumbling-window chain, never with `group_by`. The timing pass does + each output state merges the input states of its key. Over a `TimeRange` + over a tumbling-window chain it merges the windows that range selects. The timing pass does not place it yet (`ExecutionDataStateError::UnimplementedOperator`). Reserved (migrated but unimplemented; schema derivation rejects them with @@ -81,7 +81,7 @@ Exact work is represented by the ordinary operators, unchanged: - `BinaryOp` combines independently planned operands. Summary planning may set its typed division guards (`checked_finite_division`, `checked_relative_division`); the operator's timing comes from the - materialization assignment, not from the operator. + physical properties assignment, not from the operator. - Aggregate, projection, filter, sort and limit over an evaluation are the ordinary `Aggregate`, `Project`, `Filter`, `Sort` and `Limit` operators reading an ASAP node. Exact-accumulator state may pass through the projection-like @@ -108,67 +108,70 @@ happens: **ingestion time** or **query time**. Operator identity must not imply one of these phases. Backend capability restrictions are implementation gaps, not definitions of the operator. -The logical DAG carries no timing: `OperatorNode::timing` is `None` on every -front-end node and every candidate, and `map_children` clears it. +Operator nodes carry no physical properties: the logical DAG has no timing +or lifetime. Materialization ([#509 §Materialization](../proposals/planner-layering.md#materialization)) -gives each sub-DAG one of three options: `NotMaterialized`, -`MatAtIngestion(medium)` or `MatAtQuery(medium)`, with `Medium::Memory` or -`Medium::Disk`. A -[`MaterializationAssignment`](../../../crates/types/src/ir/properties/timing.rs) records -the option per sub-DAG, named by the node whose output would be stored. Any -node may be named: a summary state, a population, or, as in #509 Example 4, -the tumbling windows a query reads. `set(node, materialization)` assigns -one, and every other sub-DAG is `NotMaterialized`. Choosing the options is -Stage 2's job (#604); until then the planner materializes nothing. -`apply_materialization_timings(root, &assignment, &mut NodeToTimedNodeCache)` then writes a -timing and a materialization into every node, top-down: +decides, for each sub-DAG, whether its output is materialized, and if so with which +[`PhysicalProperties`](../../../crates/types/src/stages/physical/properties.rs): a +timing and a `Lifetime` (#1200). The lifetime is `None` (discarded after its +reader), `OneEvaluation(medium)` (materialized until every reader of one evaluation +finishes) or `AcrossEvaluations(medium)` (materialized for later evaluations), with +`StorageMedium::Memory` or `StorageMedium::Disk`; an ingestion-time output cannot be materialized +for `OneEvaluation`. A +[`PhysicalPropertiesAssignment`](../../../crates/types/src/ir/properties/timing.rs) records, +per sub-DAG, the properties of its materialized output, named by the node whose output +would be materialized. Any node may be named: a summary state, a population, or, as in +#509 Example 4, the tumbling windows a query reads. `set(node, properties)` +lists one; an entry whose lifetime is `None` fixes only the node's timing, and every +sub-DAG the assignment does not list is not materialized. Stage 2 chooses the +assignment. Operator nodes carry no timing or lifetime (#1200): physical +stages read them from the assignment. +`apply_physical_properties(root, &assignment)` then completes it into a timing and a +lifetime for every node, top-down, and returns the completed assignment: - `SummaryEstimate` and `EvaluatePopulation` run at query time; -- a node assigned `MatAtIngestion` runs at ingestion time, and one assigned - `MatAtQuery` at query time; a `SummaryAgg` or `MaintainPopulation` that is - `NotMaterialized` runs at query time. Every node upstream of an - ingestion-time node also runs at ingestion time, so `MatAtIngestion` over - something that can only exist at query time (an evaluation, or a state - that is not `MatAtIngestion`) fails with +- a node the assignment lists runs at its entry's timing; a `SummaryAgg` + or `MaintainPopulation` that is not materialized runs at query time. Every node + upstream of an ingestion-time node also runs at ingestion time, so a materialized + ingestion-time output over something that can only exist at query time (an + evaluation, or a state not materialized at ingestion time) fails with `ExecutionDataStateError::UpstreamNotAtIngestion` ("upstream of ingestion-time node not at ingestion time"); -- every other node is not stored and runs when its consumer runs: everything +- every other node is not materialized and runs when its consumer runs: everything that feeds an ingestion-time node runs at ingestion time, everything above - an evaluation at query time. A query-time consumer reads a `MatAtIngestion` - node's stored output. - -A node whose timing a candidate builder fixed (the query-time -`FinalizeExactAccumulator` read boundary of `finalize_query_candidate`) keeps -it: an explicit assignment that agrees with that timing is kept, and one that -needs the other timing fails with -`ExecutionDataStateError::MaterializationConflictsWithFixedTiming`; an -explicit `MatAtIngestion` is never silently moved or dropped. A root may be -`MatAtIngestion`, and the query reads its stored output; a root of rows that + an evaluation at query time. A query-time consumer reads the stored output + of a node materialized at ingestion time (`AcrossEvaluations`). + +Applying a completed assignment again gives every node the same properties: the +facade re-applies Stage 3's selection this way for export +(`PlanOutput::physical_properties`). A root may be materialized +at ingestion time, and the query reads its stored output; a root of rows that runs at ingestion time without being stored there fails with `ExecutionDataStateError::UnstoredIngestionRoot`. The pass then validates every edge (rows or exact-accumulator state into a -`SummaryAgg`, state into an evaluation, a timed `MaintainPopulation` under +`SummaryAgg`, state into an evaluation, a `MaintainPopulation` under an `EvaluatePopulation`, no ingestion work reading a query-time value) and rejects a node reached from two consumers that need different timings; `split_shared_by_timing` copies such a sub-DAG for one side before the -assignment is applied. How long a materialized output is kept (retention) is -also Stage 2's decision and is not in the assignment. - -`MaterializationAssignment::with_default(materialization)` gives every sub-DAG -that is not set explicitly `materialization` where the DAG allows it; a -`MatAtIngestion` default over a sub-DAG that can only run at query time leaves -that sub-DAG `NotMaterialized` instead of failing (the default applies to -`SummaryAgg` and `MaintainPopulation` nodes). The test helpers `validate_at_ingestion` and -`planned_data_state` (feature `test-support`) use it to check, at planning time and without keeping -anything, that a candidate stays valid when every sub-DAG is `MatAtIngestion`. +assignment is applied. How much a materialized output retains (retention) is decided +later and is not in the assignment. + +A node the assignment does not set is not materialized. The test helper +`PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed(roots)` +(feature `test-support`) sets every `SummaryAgg`, `SummaryTumblingWindowChain` +and `MaintainPopulation` materialized at ingestion time where the DAG allows it (not +filtered, nothing below it that can only run at query time). The test +helpers `validate_at_ingestion` and `planned_data_state` use it to check, at +planning time and without materializing anything, that a candidate stays valid +when every sub-DAG is materialized at ingestion time. ## Exported DAG The pre-ASAP DAG and the post-ASAP DAG are both logical: they describe what is computed, not which physical operators execute it. Planning builds and shares `OperatorNode` trees; -[`asap_types::ir::physical_export::compile_physical_asap_dag`](../../../crates/types/src/ir/physical_export.rs) +[`asap_types::plan::physical_asap_dag::compile_physical_asap_dag`](../../../crates/types/src/plan/physical_asap_dag.rs) converts a selected, timed tree into a `PhysicalASAPDAG` with stable node IDs and typed edges (`compile_physical_asap_workload` does the same for a batch, with one root per query). The plan file a runtime installs, @@ -188,17 +191,20 @@ children as edges and no embedded sub-DAGs: expressions (`scalar(v)`). Every edge records the intermediate schema, the producer's data state and grouping/window compatibility. - Each node records `output_state` (timing plus `Raw` or `SummaryState`), - `output_schema` and `guarantee`. Export reads the timing written by - `apply_materialization_timings` and rejects an untimed node + `output_schema` and `guarantee`. Export reads each node's timing and lifetime + from the assignment `apply_physical_properties` completed and rejects a node + it does not list (`ExecutionDataStateError::UntimedNode`); it does not re-run data-state validation. -Each node also records its `materialization`, including the medium, so the -deployment knows which outputs to store and where; validation rejects a -`SummaryAgg` or `MaintainPopulation` whose timing differs from the one its -materialization implies, and any `MatAtIngestion` or `MatAtQuery` node at the -other timing. Ingestion work cannot depend on a future query result. The -deployment executes the plan as given and does not change its materialization +Each node also records its `lifetime`, including the medium, so the +deployment knows which outputs to materialize, for how long and where; validation +rejects a timing and lifetime that cannot go together (an ingestion-time +output materialized for `OneEvaluation`) and a `SummaryAgg` or `MaintainPopulation` +that is not materialized outside query time +(`PhysicalASAPDAGValidationError::PhysicalProperties`). Ingestion +work cannot depend on a future query result. The +deployment executes the plan as given and does not change its lifetime or timing ([#509 Assumption 8](../proposals/planner-layering.md#assumptions)); it must separately check that it has an implementation and a valid data source for each placement. @@ -231,7 +237,7 @@ failure probabilities are combined, and the score guarantee remains in the membership guarantee's child provenance. An exact request does not accept this approximate output path merely because its selected identities are certified. -The materialization assignment decides whether these summaries run at +The physical properties assignment decides whether these summaries run at ingestion time or at query time, and the exported DAG carries that placement. Either placement must give each evaluation a complete rate window and an isolated summary state, or maintain an equivalent replacement diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index cf597bf16..3eb03b8c5 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -39,22 +39,21 @@ The Logical Post-ASAP DAG is preceded by the Pre-ASAP DAG, the language-independent query semantics before summary selection. Both are logical `OperatorNode` DAGs. `compile_physical_asap_dag` exports the selected DAG as a `PhysicalASAPDAG`, which is the Physical Plan Compiler's input. Its per-node execution phase (ingestion or query time) is -decided by a `MaterializationAssignment`, as the layer contract below states. +decided by a `PhysicalPropertiesAssignment`, as the layer contract below states. ### Layer contract 1. **Logical Post-ASAP** (`CandidateLogicalASAPDAGs`) decides what to compute: summary families, readouts and sharing. It does not decide placement; timing that a realization strategy writes while building a candidate is provisional. -2. **Materialization** (Stage 2, #509) chooses `NotMaterialized`, - `MatAtIngestion(medium)`, `MatAtQuery(medium)` or - `MatAtQueryKeptAcrossEvaluations(medium)` for each sub-DAG whose - output may be stored (a `SummaryAgg` or a `MaintainPopulation`) and records - it in a `MaterializationAssignment`. `apply_materialization_timings` writes - every node's `ExecutionTiming` and `Materialization`: a `MatAtIngestion` - state and all of its inputs run at ingestion time; readouts, other - consumers, and the other states run at query time. The default assignment - is `NotMaterialized` everywhere; `PlanOutput::physical_asap_dag` applies it +2. **Materialization** (Stage 2, #509) chooses, for each sub-DAG whose + output may be stored (a `SummaryAgg` or a `MaintainPopulation`), whether + it is materialized and with which timing and `Lifetime` (`None`, + `OneEvaluation(medium)` or `AcrossEvaluations(medium)`), and records it in + a `PhysicalPropertiesAssignment`. `apply_physical_properties` writes every + node's `ExecutionTiming` and `Lifetime`: a state materialized at ingestion time and + all of its inputs run at ingestion time; readouts, other consumers, and the + other states run at query time. The default assignment materializes nothing; `PlanOutput::physical_asap_dag` applies it only to nodes the optimization pass left untimed (a pass's own timing, such as Stage 3's ingestion-time tumbling windows, is kept). 3. **Physical compile** (executor) reads the plan's materialized views: each @@ -97,7 +96,7 @@ and a feasible candidate that loses on cost. Absence is not a cost comparison. For `sum by(job)(rate(m[1m]))`, Rate remains per series before grouped Sum. `CandidateLogicalASAPDAGs` offers one such candidate, with a per-series Rate state and a grouped -Sum state. Its materialization assignment places it: an ingestion-time Sum state +Sum state. Its physical properties assignment places it: an ingestion-time Sum state finalizes Rate and builds Sum within a bounded precompute run; a query-time Sum over an ingestion-time Rate state leaves the Rate readout and Sum in the query DAG. Storing a value requires its exact evaluation window, revision, readiness and serving @@ -113,7 +112,7 @@ Planner's candidate space decides what to compute, not placement. For an instant-vector PromQL TopK, Planner resolves rows that carry the complete series identity and lists the current-series heap realizations per root with the other candidates, unranked. Precompute or query-time placement of Rate and grouped Sum -is not a separate Planner candidate: the materialization assignment sets +is not a separate Planner candidate: the physical properties assignment sets each node's timing, and the physical compiler reads it. This is the target ownership contract. A backend path that still reconstructs @@ -473,8 +472,8 @@ These derivations are methods on `PhysicalASAPDAG` in `asap-types`, so a runtime needs no dependency on the planner crate. A **materialized view** is a node whose output is kept across evaluations -(`MatAtIngestion` or `MatAtQueryKeptAcrossEvaluations`). A `MatAtQuery` node is -not a view: its output is discarded after one evaluation. Everything a runtime +(lifetime `AcrossEvaluations`, at ingestion or at query time). A node materialized for +`OneEvaluation` is not a view: its output is discarded after one evaluation. Everything a runtime needs that the DAG determines is derived from the DAG, so the plan file has no second copy to keep consistent. Stage 3's selection report is returned beside the plan, not inside it. @@ -621,7 +620,7 @@ The refresh window is derived from the DAG alone (time ranges, shifts and tumbli widths), like the views and their retention. Sealing, lateness, threading and the alignment of the window grid stay the deployment's. -Stage 3's cost assumes this grouping (`asap_cost::execution_demand::computations`): +Stage 3's cost assumes this grouping (`asap_cost::plan::execution_demand::computations`): a stateless ingestion-time node is charged once per refresh window of the views above it, not once per view. A runtime that splits a group (for example, because its sources seal at different times) computes the same diff --git a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md index 06969a5bb..ed5f8a9e7 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md @@ -1,7 +1,7 @@ # Analytical resource cost model > Status: implemented model with explicit support limits. The -> [analytical estimator](../../../../crates/cost/src/analytical.rs) +> [analytical estimator](../../../../crates/cost/src/node/analytical.rs) > is what Stage 3 measures resource usage with. The physical-plan adapter and > the legacy `CostModel` described below have been removed; Stage 3's cost > models 1–3 replace them. diff --git a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md index 0975ba578..ffb87c859 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md +++ b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md @@ -79,9 +79,9 @@ input `EvaluatePopulation` runs at query time. `MaintainPopulation` has no fixed timing: like a `SummaryAgg`, it follows its materialization -(`NotMaterialized`, `MatAtIngestion(medium)` or `MatAtQuery(medium)`, #509 +(`Lifetime::None`, `(IngestionTime, AcrossEvaluations(medium))` or `(QueryTime, OneEvaluation(medium))`, #509 §Materialization), which Stage 2 chooses; it runs at ingestion time only when -it is `MatAtIngestion`. +it is ingestion-time `AcrossEvaluations`. The rule examines compatible workload roots, sets `max_k` to the largest requested k and enables quantile readout if any consumer needs it. It emits a candidate for diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index ee15ead49..a6b298c5f 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -68,16 +68,18 @@ Based on our requirement, each field should contain the following information. 1. **What type of the ASAP Primitive is.** The field's type is a [`FieldDataType`](#63-schema-and-field-types-cratestypessrcirschemamodrs) (`crates/types/src/ir/schema/mod.rs`). A column is either a raw value or a summary state: - **Raw value**: `Plain(DataType)`, e.g., a number or a string. - - **Summary state**: described from coarse to fine by three levels: + - **Summary state**: described from coarse to fine by these terms: - | Level | Code | Values | KLL example | + | Term | Code | Values | KLL example | |---|---|---|---| | Family | `FieldDataType` variant | `ExactAggregate`, `Sketch`, `SummarySample`, `Wavelet`, `StatModel` | `Sketch` | - | Algorithm | `SketchAlgorithm` (other families: `ExactKind`, `SamplingKind`, …) | `Kll`, `Cms`, `Hll`, `DDSketch`, … | `Kll` | - | Parameters | `SketchParams` (other families: `ExactParams`, `SamplingParams`, …) | per algorithm | `Kll { k: 200 }` | + | Algorithm | `SketchAlgorithm` (other families: `ExactKind`, `SamplingKind`, …) | `Kll`, `Cms`, `Hll`, `DDSketch`, `HydraKll`, … | `Kll` | + | Summary type | family and algorithm: one `FieldDataType` value, the column's type | `FieldDataType::Sketch(SketchAlgorithm)`, … | `Sketch(Kll)` | + | Parameters θ | `SketchParams` (other families: `ExactParams`, `SamplingParams`, …), on the candidate's `LogicalProperties`, not in the type | per algorithm, from `parameter_list(algorithm)` | `Kll { k: 547 }` | - - For a sketch, algorithm and parameters are bundled as one `SketchKind` ([§6.4](#64-summary-family-parameters-cratestypessrcirschemasketchrs)). A sketch also records its `GroupingStrategy`: one instance per group, or one shared structure (Hydra) for all groups. - - So a quantile KLL sketch with `k = 200`, one instance per group, has the type `Sketch(SketchKind { Kll, Kll { k: 200 } }, PerSubpopulationInstance)`. Which statistics a family answers is the family-capability table's (`FamilyCapability`), not part of the type. + - The summary type is the operator's `summary_type` argument (`SummaryAgg`, the chain). θ is structure only (sizes, no sized-for error fields) and lives on the candidate, so builds that differ only in θ are candidates of one memo group. + - A shared multi-subpopulation structure (Hydra) is an algorithm of its own (`HydraKll`, `HydraCms`, `HydraCountSketch`, …), not a separate grouping dimension. + - So a quantile KLL sketch with `k = 547` has the type `Sketch(Kll)` and θ `Kll { k: 547 }`. Which statistics a summary type answers is its row of the capability table (`SummaryTypeCapability`), not part of the type. 2. **What query intent the summarized ASAP Primitive can support.** This is not stored in the field: it follows from the type in item 1. There are two kinds of intent: @@ -92,7 +94,7 @@ Based on our requirement, each field should contain the following information. | `FrequencyL2`, `FrequencyEntropy` | `Sketch`: `UnivMon` | `SummaryEstimate(FrequencyL2 \| FrequencyEntropy)` | | `Sum`, `Count`, `Min`, `Max`, `Rate`, `IRate`, `Increase` (exact) | `ExactAggregate(ExactKind, …)` | `FinalizeExactAccumulator` | - The sketch candidates are `summary_candidates(intent)` in `crates/logical-asap-optimization/src/pass1/realization.rs`; exact accumulators come from Pass 1's `ExactAccumulator` candidates in `pass1/per_computation_candidates.rs`; the evaluations are `SketchStatistic` ([§6.5](#65-update-input-and-evaluations-irschemasketchrs-iroperatormaintained_populationrs)). + The sketch candidates are `summary_candidates(intent)` in `crates/logical-asap-optimization/src/pass1/realization.rs`; exact accumulators come from Pass 1's `ExactAccumulator` candidates in `pass1/computation_targets.rs`; the evaluations are `SketchStatistic` ([§6.5](#65-update-input-and-evaluations-summary_typestatisticrs-iroperatormaintained_populationrs)). - **Time window aggregation intents**: whether states built over smaller windows can answer a larger one. This depends on how the family combines states: - **Merge** (`SummaryMerge`, §5.6): states over disjoint tumbling windows combine into the state of their union, e.g. two 1-minute KLL states answer a 2-minute quantile. Requires a mergeable family. @@ -296,7 +298,7 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` **When two definitions are equal.** Merging (§5.6) requires equal definitions. -- They must have the same structure. Planning details are ignored: `timing`, `guarantee` and `coverage_cache`. So a tumbling window built at ingestion time can merge with one built at query time. +- They must have the same structure. Planning details are ignored: `guarantee` and `coverage_cache`. Operator nodes carry no timing (#1200), so a tumbling window built at ingestion time can merge with one built at query time. - `SummaryUpdate.weight_domain` is compared too. It is computed from the rest, so it differs only if something is wrong. - States over different tables never merge: a KLL over `m1` and one over `m2` have different definitions. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. @@ -665,11 +667,11 @@ With an exact target, the planner can keep all latencies per job in one maintain - **Implemented.** All children must be state with exactly one state field and identical schemas. - **Coverage.** It also requires equal `definition`s and selections that do not overlap, and computes the merged coverage. -- **Newest tumbling windows:** an optional `newest_tumbling_windows: n` merges only the newest `n` tumbling windows of the one `SummaryTumblingWindowChain` the merge reads (`1 ≤ n <` its tumbling-window count). Its coverage is one build over the time range those tumbling windows cover, so one chain kept for the longest window serves every query whose window, on the same tumbling windows, ends at the same time and is shorter. -- **Rollup:** an optional `group_by` field, so one operator does both merge and rollup. It lists the input's group columns the output is keyed by; `None` keeps the input's groups. A rollup merges every input state, so it does not take `newest_tumbling_windows`; it reads a merge of the newest tumbling windows instead: +- **Chain windows:** a merge over a `TimeRange` (and `TimeShift`s) over a `SummaryTumblingWindowChain` merges the windows that range selects; there is no window-selection field. Its coverage is that of one build over the same `TimeRange` and `TimeShift`s, so one chain serves every query over the same input whose range and offset lie on its windows. +- **Rollup:** an optional `group_by` field, so one operator does both merge and rollup. It lists the input's group columns the output is keyed by; `None` keeps the input's groups. A rollup merges every input state: ```rust -SummaryMerge { children: Vec, newest_tumbling_windows: Option, group_by: Option> } +SummaryMerge { children: Vec, group_by: Option> } ``` **Example A: time tumbling windows.** p99 of each series of metric `m` over the last 2 minutes: @@ -822,7 +824,7 @@ input groups output groups | Question | Answer | |---|---| | What comes out? | the children's schema; with `group_by`, its columns in order, then the state column. A `group_by` of no columns over a SQL relation's states stands for a global SQL aggregate, which has one row over no input rows: it outputs one state, empty when there is no input state | -| When is it rejected? | no children; a child is not state; the schemas differ (e.g. different sketch parameters); the definitions differ (different column, filters or source); the selections may overlap (#646; with #592, only where the family does not allow it); the family cannot merge at all (#592); a `group_by` column is not a group key of the children's build, appears twice, or is the state or time column; `group_by` over a family that cannot merge different groups (counter states, `IRate`, Theta, Hydra); `group_by` together with `newest_tumbling_windows` | +| When is it rejected? | no children; a child is not state; the schemas differ (e.g. different sketch parameters); the definitions differ (different column, filters or source); the selections may overlap (#646; with #592, only where the family does not allow it); the family cannot merge at all (#592); a `group_by` column is not a group key of the children's build, appears twice, or is the state or time column; `group_by` over a family that cannot merge different groups (counter states, `IRate`, Theta, Hydra) | | What is its coverage? | the shared `definition` (with `group_by`, the same build grouped by the columns `group_by` names), and the union of the children's selections. Two boxes join when they differ in only one dimension and their union is again one constraint, as in the paper's one range per column: touching time windows (`(−2m, −1m]` and `(−1m, 0]` give `(−2m, 0]`), touching value ranges (`latency < 100` and `latency >= 100` give `latency ∈ (−∞, ∞)`, which still excludes NULL), or value sets of one column (`{us}` and `{eu}` give `{us, eu}`). Gaps stay as separate boxes | | State or value? | state in, state out | @@ -877,15 +879,15 @@ Every struct field and enum-variant field below has a comment saying what it hol OperatorNode §6.1 ├── operator: Operator::ASAP(ASAPOp) §6.2 │ ├── SummaryAgg -│ │ ├── family: FieldDataType ───────────┐ §6.3 → §6.4 +│ │ ├── summary_type: FieldDataType ─────┐ §6.3 → §6.4 │ │ └── input: SummaryUpdate │ §6.5 (what each row adds) │ ├── SummaryEstimate.query: SketchStatistic │ §6.5 (what is read out) │ ├── MaintainPopulation.population │ §6.5 │ └── EvaluatePopulation.evaluation │ §6.5 ├── schema: Schema │ §6.3 │ └── fields[i].dtype: FieldDataType ◀───────┘ §6.3 (the same type: the output -│ └── Sketch(SketchKind, GroupingStrategy) `state` field has type `family`) -│ └── algorithm, params §6.4 +│ └── Sketch(SketchAlgorithm) `state` field has type `summary_type`) +│ (θ, `SketchParams`, is on the candidate) §6.4 ├── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) └── coverage() → SummaryCoverage §6.6 └── definition: OperatorNode (again a node, so the same structure) @@ -895,12 +897,12 @@ OperatorNode §6.1 |---|---|---| | §6.1 DAG node | `OperatorNode` | the DAG itself | | §6.2 ASAP operators | `ASAPOp` | `OperatorNode.operator` | -| §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.family` | -| §6.4 Summary-family parameters | `SketchKind`, `SketchParams`, `GroupingStrategy`, exact/sample/wavelet/model params | the payload of each non-plain `FieldDataType` variant | +| §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.summary_type` | +| §6.4 Summary algorithms and parameters | `SketchAlgorithm`, `SketchParams`, `HydraParams`, exact/sample/wavelet/model kinds and params | the payload of each non-plain `FieldDataType` variant; θ on a candidate's `LogicalProperties` | | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | | §6.6 Summary coverage | `SummaryCoverage`, `SelectionBox`, `Constraint` | `OperatorNode::coverage()` on summary nodes | -**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `family = Sketch(SketchKind::new(Kll, Kll { k: 200 }), PerSubpopulationInstance)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5). Its `schema` (§6.3) is `(job Utf8, state Sketch(KLL k=200))`: the `state` field has the same type as `family`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). +**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). ### 6.1 DAG node (`crates/types/src/ir/operator/node.rs`) @@ -921,9 +923,6 @@ pub struct OperatorNode { /// The accuracy statement of the output, once known. `None` does not /// mean exact. pub guarantee: Option, - /// When the node runs: `IngestionTime` or `QueryTime`. `None` until - /// planning assigns it. - pub timing: Option, /// Cache for `coverage()`. Filled on first use, never serialized, /// ignored by equality, emptied on clone. Not a source of truth: /// coverage can always be derived again. @@ -951,14 +950,12 @@ pub enum ASAPOp> { /// The input rows. child: C, /// The summary type of the output `state` field. Never `Plain`. - family: FieldDataType, + summary_type: FieldDataType, /// What each input row adds to the state (item and weight). input: SummaryUpdate, /// The grouping: `Reduce(by columns)`, or `PerEntity` for one state per /// input series without grouping. reduction: Reduction, - /// Whether each group gets its own sketch or all groups share one. - grouping: GroupingStrategy, /// Rows to include, applied before updating the state. `None`: all rows. filter: Option>, }, @@ -988,9 +985,6 @@ pub enum ASAPOp> { SummaryMerge { /// The states to merge; all have the same schema. children: Vec, - /// Merge only the newest `n` tumbling windows of the one `SummaryTumblingWindowChain` read; - /// `None` merges every input state. Not with `group_by`. - newest_tumbling_windows: Option, /// A rollup: the input's group columns the merged states are keyed /// by, in output order; every input state whose values in them match /// goes into one output state. `None` keeps the input's groups. @@ -1009,7 +1003,7 @@ pub enum ASAPOp> { outer: C, // one input state inner: C, // the other input state key: ColumnId, // the join key column - family: FieldDataType, // the summary type of the result + summary_type: FieldDataType, // the summary type of the result }, Extension { child: C, // the input @@ -1032,7 +1026,7 @@ impl ASAPOp { /// Whether the operator is reserved and cannot be built yet: Subtract, /// Delete, Join, Extension. pub fn is_unimplemented(&self) -> bool; - /// The summary type this operator outputs: `family` for SummaryAgg and + /// The summary type this operator outputs: `summary_type` for SummaryAgg and /// SummaryJoin, the inputs' state type for SummaryMerge, `None` otherwise. pub fn produced_state(&self) -> Option<&FieldDataType>; /// The output schema, derived from the operator and its inputs. An error @@ -1111,15 +1105,16 @@ impl Field { pub fn is_plain(&self) -> bool; } -/// A column's type: a plain value, or summary state of one family (§3). +/// A column's type: a plain value, or summary state of one summary type +/// (family and algorithm, §3). pub enum FieldDataType { /// A readable value of this type. Plain(DataType), /// Exact accumulator state: which accumulator, and its parameters. ExactAggregate(ExactKind, ExactParams), - /// Sketch state: the chosen sketch (algorithm, parameters), and - /// whether each group has its own sketch or all groups share one. - Sketch(SketchKind, GroupingStrategy), + /// Sketch state of one algorithm; θ is not part of the type. The + /// `Hydra*` algorithms share one structure across all groups. + Sketch(SketchAlgorithm), /// Summary sample state: the sampling method, and its parameters. #[serde(rename = "Sample")] SummarySample(SamplingKind, SamplingParams), @@ -1149,7 +1144,7 @@ pub enum DataType { } ``` -### 6.4 Summary-family parameters (`crates/types/src/ir/schema/sketch.rs`) +### 6.4 Summary algorithms and parameters (`crates/types/src/summary_type/parameters.rs`) ```rust /// Which exact accumulator. None of them has parameters, so `ExactParams` @@ -1157,28 +1152,15 @@ pub enum DataType { pub enum ExactKind { Sum, Count, Min, Max, Increase, Rate, IRate } pub enum ExactParams { Sum, Count, Min, Max, Increase, Rate, IRate } -/// A chosen sketch. Built only through `new`, so the two fields always agree. -/// Which statistics it answers is its family's row of the family-capability -/// table (`FamilyCapability`). -pub struct SketchKind { - /// Which sketch algorithm. - algorithm: SketchAlgorithm, - /// That algorithm's size parameters; must match `algorithm`. - params: SketchParams, -} -impl SketchKind { - /// The only constructor. Panics if `params` belong to a different - /// algorithm. - pub fn new(algorithm: SketchAlgorithm, params: SketchParams) -> Self; - /// The sketch algorithm. - pub fn algorithm(&self) -> &SketchAlgorithm; - /// The algorithm's size parameters. - pub fn params(&self) -> &SketchParams; -} -// Theta is not offered as a candidate until asap_sketchlib has a Theta sketch. -pub enum SketchAlgorithm { UnivMon, Kll, Cms, Hll, DDSketch, CmsWithHeap, Kmv, Theta, CountSketch, CountSketchWithHeap } +/// `asap_sketchlib`'s algorithm enum, the Hydra algorithms (`HydraKll`, +/// `HydraCms`, `HydraCountSketch`, …) included. Which statistics each +/// answers is its row of the capability table (`SummaryTypeCapability`). +pub use asap_sketchlib::SketchAlgorithm; -/// Size parameters of each algorithm. Larger values: more memory, less error. +/// θ: the structure of each algorithm, from its predefined list +/// (`parameter_list`). Larger values: more memory, less error. No field +/// records what error a configuration was sized for; equality compares +/// every field. pub enum SketchParams { /// `layers`: number of sampling levels. Each level has a Count Sketch of /// `sketch_rows` hash rows × `sketch_cols` counters, and a heap of the @@ -1196,24 +1178,15 @@ pub enum SketchParams { CmsWithHeap { width: u32, depth: u32, heap_size: u32 }, /// `k`: number of smallest hash values kept. Kmv { k: u32 }, - /// `k`: number of hash values kept (nominal entries). - Theta { k: u32 }, /// `width`: counters per row. `depth`: number of hash rows. CountSketch { width: u32, depth: u32 }, /// Count Sketch `width` × `depth`, plus a heap of the `heap_size` heaviest items. CountSketchWithHeap { width: u32, depth: u32, heap_size: u32 }, + /// θ of a `Hydra*` algorithm. + Hydra(HydraParams), } -/// How a grouped sketch is laid out across the `by` groups. Independent of the family. -pub enum GroupingStrategy { - /// One separate sketch per group. The default. - PerSubpopulationInstance, - /// One shared structure for all groups (Hydra). `kind`: which shared - /// structure. `params`: its sizes. - SharedMultiSubpopulation { kind: HydraKind, params: HydraParams }, -} -pub enum HydraKind { HydraKll /* experimental, no error bound */, HydraCms, HydraCountSketch } -/// Sizes of the shared structure. +/// Sizes of a Hydra structure, one shared by all groups. pub enum HydraParams { /// `k`: KLL `k` each group sees. `shared_buckets`: size of the one structure /// shared by all groups. @@ -1224,11 +1197,6 @@ pub enum HydraParams { /// Same fields as `HydraCms`, for Count Sketch. HydraCountSketch { width: u32, depth: u32, shared_rows: u32, shared_columns: u32 }, } -/// The shared (Hydra) version of an algorithm, if one with a known error -/// bound exists: `Cms` → `HydraCms`, `CountSketch` → `HydraCountSketch`; -/// `None` for all others. -pub fn hydra_kind_for(a: &SketchAlgorithm) -> Option; - /// `size`: number of rows kept in the reservoir. pub enum SamplingKind { Reservoir } pub enum SamplingParams { Reservoir { size: u32 } } /// `coefficients`: number of wavelet coefficients kept. @@ -1237,7 +1205,7 @@ pub enum WaveletKind { Haar } pub enum WaveletParams { Haar { coeffi pub enum StatModelKind { Parametric } pub enum StatModelParams { Parametric { family: String } } ``` -### 6.5 Update input and evaluations (`ir/schema/sketch.rs`, `ir/operator/maintained_population.rs`) +### 6.5 Update input and evaluations (`summary_type/statistic.rs`, `ir/operator/maintained_population.rs`) This part of the code answers two questions about summary state: @@ -1341,7 +1309,7 @@ pub struct CurrentSeriesInput { } ``` -A finalized value's accuracy statement is `AccuracyGuarantee` (`ir/properties/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. +A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. ### 6.6 Summary coverage (`crates/types/src/ir/properties/summary_coverage.rs`) @@ -1361,11 +1329,9 @@ pub struct SummaryCoverage { pub struct SelectionBox { /// One constraint per restricted column. A column not in the map is /// unrestricted. + /// The timestamp column is one of them; there is no separate time + /// field. pub columns: BTreeMap, - /// The time window relative to the evaluation time, in ms, e.g. - /// `(Excluded(-120000), Included(-60000))` for `(−2m, −1m]`. - /// `None`: no time restriction. - pub relative_time: Option<(Bound, Bound)>, } /// Names a column across nodes, independent of its position. @@ -1384,10 +1350,31 @@ pub enum Constraint { NotIn(Vec), /// The value lies between `lower` and `upper`. Each end is `Included`, /// `Excluded` or `Unbounded`. - Interval { lower: Bound, upper: Bound }, + Interval { lower: Bound, upper: Bound }, // HashPartition { columns, of, index }: added with its first producer. } +/// An end of an interval. Limits compare only within one variant. +pub enum IntervalConstraintLimit { + /// A constant, e.g. `Float64(10.0)` or `Timestamp(ms since the epoch)`. + FixedValue(ScalarValue), + /// The evaluation time t of a repeatedly evaluated query plus this + /// many ms; only on the timestamp column. + EvaluationTimePlusMs(i64), +} +``` + +A PromQL range `m[r]` evaluated at t reads the samples in (t − r, t]: left-open, right-closed. + +| Query fragment | `lower` | `upper` | +|---|---|---| +| PromQL `m[5m]` | `Excluded(EvaluationTimePlusMs(-300_000))` | `Included(EvaluationTimePlusMs(0))` | +| PromQL `m[5m] offset 1h` | `Excluded(EvaluationTimePlusMs(-3_900_000))` | `Included(EvaluationTimePlusMs(-3_600_000))` | +| PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | +| SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(..)))` | `Unbounded` | +| SQL `WHERE price > 10` | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | + +```rust impl SummaryCoverage { /// Computes the coverage of a summary node from its sub-DAG (§4.2.2). /// Errors: `NotSummary` for a node that is not a `SummaryAgg`, diff --git a/docs/design_docs/proposals/joint-planning-implementation-plan.md b/docs/design_docs/proposals/joint-planning-implementation-plan.md index e561406ac..a6cb40d31 100644 --- a/docs/design_docs/proposals/joint-planning-implementation-plan.md +++ b/docs/design_docs/proposals/joint-planning-implementation-plan.md @@ -12,18 +12,18 @@ Preserve supported accuracy semantics, physical eligibility, cost objectives, sc | Scope | Current implementation | Responsibility | | --- | --- | --- | -| Logical identity | `logical-asap-optimization/src/stage0.rs`; `types/src/stages/` | `DagOfLogicalMemoGroups`, `MemoGroup` (`asap_types::stages`; built by `generate_dag_of_logical_memo_groups`), stable initial query/group references. | +| Logical identity | `logical-asap-optimization/src/stage0.rs`; `types/src/stages/` | `DagOfLogicalMemoGroups`, `MemoGroup` (`asap_types::stages`; built by `memo_of_queries`), stable initial query/group references. | | Computation and sharing rules | `logical-asap-optimization/src/pass2/memo/{candidates,rules,state_groups,shared_configurations,window_candidates}` | Populate groups, retain local candidates, expose shared states and compatible rewrites. | -| Physical legality | `physical-asap-optimization` | Executable operators, timing, materialization eligibility, and `PhysicalWorkloadPlan`. | -| Candidate dependency inventory | `physical-asap-optimization/src/catalog`; `types/src/stages/physical` | Local `PhysicalCandidate`, `ProducerRequirement` (`asap_types::stages`), compatible producers, structural pruning, the selected DAG (`DagOfSelectedCandidates`, one node per run of a candidate), and reconstruction. Stage 2 makes no execution-scope copies; today's Stage 3 engine makes its own (`plan-selection/src/search/candidates/scopes.rs`). | +| Physical legality | `physical-asap-optimization` | Executable operators, timing and materialization eligibility. | +| Candidate dependency inventory | `physical-asap-optimization/src/catalog`; `types/src/stages/physical` | Local `PhysicalCandidate`, `ProducerRequirement` (`asap_types::stages`), compatible producers, structural pruning; Stage 3 builds the selected plan (`PhysicalASAPDAG`) directly. Stage 2 makes no execution-scope copies; today's Stage 3 engine makes its own (`plan-selection/src/search/candidates/scopes.rs`). | | Solver | `plan-selection/src/search/milp` | Activation, producer/root assignment, reachability, acyclicity, resource constraints, and solver warm starts. | | Search and validation | `plan-selection/src/search/refinement` | Actual evaluator checks, sound cuts, objective-specific ranking, and certified component splitting. | -| Pricing support | `plan-selection/src/search/costing` | Certified local bounds and consistent calibration. Production execution demand is owned by `cost/src/execution_demand.rs`; the search demand reference is test-only. | +| Pricing support | `plan-selection/src/search/costing` | Certified local bounds and consistent calibration. Production execution demand is owned by `cost/src/plan/execution_demand.rs`; the search demand reference is test-only. | | Pipeline and lifecycle | `plan-selection/src/search/{pipeline,route}` | Initial groups, generation accounting, seed/full search budgets, archives, reports, and selected workload outputs. | -`PhysicalCandidate` is a local selectable computation. `PhysicalWorkloadPlan` is a complete Stage 2 workload DAG; they are not interchangeable. Physical candidate configuration in selection invokes the authoritative Stage 2 legality code rather than duplicating its rules. Logical rules do not depend on the solver crate. +`PhysicalCandidate` is a local selectable computation; the plan (`PhysicalASAPDAG`) is the complete workload DAG Stage 3 builds; they are not interchangeable. Physical candidate configuration in selection invokes the authoritative Stage 2 legality code rather than duplicating its rules. Logical rules do not depend on the solver crate. -Pass 1 has one entry, `generate_per_computation_logical_memo_candidates`, which populates the already-created memo groups (its target list, `PerComputationCandidates`, is a crate-private working form). This is not a second workload selector. Pass 2 can add intermediate state groups and consumer candidates; sharing a component does not merge distinct requested results: +Pass 1 has one entry, `pass1_memo`, which populates the already-created memo groups (its target list, `PerComputationCandidates`, is a crate-private working form). This is not a second workload selector. Pass 2 can add intermediate state groups and consumer candidates; sharing a component does not merge distinct requested results: ```text G95 = { ExactP95, LocalKLLQuantileP95, EstimateP95(G_KLL) } diff --git a/docs/design_docs/proposals/joint-planning-search.md b/docs/design_docs/proposals/joint-planning-search.md index 713a7e356..2558a5341 100644 --- a/docs/design_docs/proposals/joint-planning-search.md +++ b/docs/design_docs/proposals/joint-planning-search.md @@ -125,17 +125,17 @@ G_KLL += MergeLatest15(G_tumbling_windows) Non-summary query-time materialization must be a query output or have a selected immediate consumer used by only one query; the materialized node itself must have multiple query readers. These connection-dependent constraints match Stage 2's highest-shared-node rule and are enforced in MILP. Summary-state eligibility remains separate. -Physical rules provide executable operators, configurations, and materialization assignments for the generated computations. For example, a supported tumbling window implementation can maintain state at ingestion and merge it at evaluation. This introduces two timing decisions within the same query, rather than one materialization choice for the whole query. Eligibility that depends on consumers is checked against the selected connections; considering every potential consumer at once could incorrectly reject a useful candidate. +Physical rules provide executable operators, configurations, and physical properties assignments for the generated computations. For example, a supported tumbling window implementation can maintain state at ingestion and merge it at evaluation. This introduces two timing decisions within the same query, rather than one materialization choice for the whole query. Eligibility that depends on consumers is checked against the selected connections; considering every potential consumer at once could incorrectly reject a useful candidate. For the running example, assume evaluations occur at aligned minute boundaries and the deployment supports ingestion-time tumbling window maintenance. Stage 2 can expand the state groups as follows; `s` is a generated KLL capacity: ```text -Group Physical candidate Computation Materialization -G_tumbling_windows P_ingest(s) Maintain aligned 1m tumbling windows MatAtIngestion(Memory) -G_KLL K_merge(s) MergeLatest15(G_tumbling_windows) MatAtQuery(Memory) -G_KLL K_build(s) BuildKLL(G_input, 15m) MatAtQuery(Memory) -G95 E95 EstimateQuantile(.95, G_KLL) NotMaterialized -G99 E99 EstimateQuantile(.99, G_KLL) NotMaterialized +Group Physical candidate Computation (timing, lifetime) +G_tumbling_windows P_ingest(s) Maintain aligned 1m tumbling windows (IngestionTime, AcrossEvaluations(Memory)) +G_KLL K_merge(s) MergeLatest15(G_tumbling_windows) (QueryTime, OneEvaluation(Memory)) +G_KLL K_build(s) BuildKLL(G_input, 15m) (QueryTime, OneEvaluation(Memory)) +G95 E95 EstimateQuantile(.95, G_KLL) (QueryTime, None) +G99 E99 EstimateQuantile(.99, G_KLL) (QueryTime, None) ``` `P_ingest(s)` retains the tumbling windows needed to cover the 15m window. `K_merge(s)` constructs window state from those tumbling windows at evaluation; `K_build(s)` instead builds window state from available raw input. Each candidate records its input requirements and costs. The estimators can use either compatible state candidate. Exact and local KLL candidates from Pass 1 also receive supported physical implementations and remain available. Stage 2 has generated choices; it has not yet selected a build or connected an estimator to it. @@ -189,7 +189,7 @@ The following cases make the accounting concrete. Let `B` be the cost of one who | Repeated common evaluations | Both queries evaluate together `n` times; state is discarded after each evaluation. | `n × (B + E95 + E99)`. Selecting one candidate does not mean building only once over the entire horizon. | | Different evaluation timestamps | Q1 evaluates every minute; Q2 every five minutes on the same grid. Both use a 15m window. | Over 15 evaluation ticks, a shareable query-local build executes 15 times, with 15 P95 extractions and 3 P99 extractions: `15B + 15E95 + 3E99`. This assumes sharing at the three common timestamps; an earlier window state cannot automatically answer a later window. | | Ingestion-maintained tumbling windows | One tumbling-window chain feeds a shared query-time merge and both estimators. | Ingestion updates and tumbling-window retention are charged once for the chain. Add `n × (M + E95 + E99)` for `n` common evaluations, plus query-local merge memory. | -| State retained across evaluations | An eligible tumbling-window-chain candidate uses `MatAtQueryKeptAcrossEvaluations(Memory)`. | Charge initial construction, required refresh/update work, retained memory/storage over its lifetime, and evaluation work. Reuse does not eliminate freshness maintenance; generate this case only when supported. | +| State retained across evaluations | An eligible tumbling-window-chain candidate uses `(QueryTime, AcrossEvaluations(Memory))`. | Charge initial construction, required refresh/update work, retained memory/storage over its lifetime, and evaluation work. Reuse does not eliminate freshness maintenance; generate this case only when supported. | | Multiple selected candidates in one group | Some consumers use a small KLL state and others a separate large state. | Charge each selected state's build/update and retention, then each consumer's local work. Equal group identity does not collapse two physical candidates into one charge. | | Shared raw-data retention | Two consumers need 5m and 15m of the same raw source and compatible retention scope. | Both consumers connect to `Retain15m` in `G_raw[d]`. Charge that candidate once; reads and intermediate KLL state have separate owners. | | Per-request overhead | Two requests share computation but still require separate response handling. | Report fixed overhead as `Σ_q N_q × h_q`; omit this constant from the optimization objective but retain its applicable resource usage. Decision-dependent overhead stays in the optimization model. | diff --git a/docs/design_docs/proposals/operator-sharing.md b/docs/design_docs/proposals/operator-sharing.md index 5f9deb85f..cb822d7a6 100644 --- a/docs/design_docs/proposals/operator-sharing.md +++ b/docs/design_docs/proposals/operator-sharing.md @@ -1,6 +1,9 @@ # Sharing Operators Between Pre-ASAP IR and Post-ASAP IR > Status: proposal, not implemented. Audience: planner designers and architects. +> Note: since #1200, operator nodes carry no `timing`; physical stages keep each +> node's timing and lifetime in a `PhysicalPropertiesAssignment`, and +> `validate_execution_timing` no longer exists. The `timing` text below predates that. > Addresses [#468](https://github.com/ProjectASAP/ASAPPlanner/issues/468). > Companion: [Decoupling operators from scalar expressions](decoupling_op_and_expr.md). diff --git a/docs/design_docs/proposals/planner-layering-example1-acceptance.md b/docs/design_docs/proposals/planner-layering-example1-acceptance.md index c66cd3aa6..0dd10ff90 100644 --- a/docs/design_docs/proposals/planner-layering-example1-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example1-acceptance.md @@ -19,7 +19,7 @@ whose expectation needs a feature that is not in the stack yet is |---|---|---| | Query-language frontends | 1 workload `LogicalDAG` | same | | Logical ASAP-aware optimization | Pass 1 (3) × identical-expression rule (2) × window-composition rule (3 × 3) = **54** | Pass 1 (3), each with 2 identical-expression candidates and 6 window-composition candidates (Q1's rate and Q2's option in six 10-s tumbling windows, alone or both, each also with a shared scan) = **27**. Sliding windows are not implemented. | -| Physical ASAP-aware optimization | Materialization options per window form = **156** | Physical operator implementation, plus materialization: each logical candidate with every sub-DAG `NotMaterialized` (27: the doc's Raw plans, with tumbling windows rebuilt at every refresh), each tumbling-window chain also `MatAtIngestion(Memory)` (30 more), and each shared input built once per evaluation, `MatAtQuery(Memory)`, with every tumbling-window chain at query time (15 more) = **72** | +| Physical ASAP-aware optimization | Physical properties options per window form = **156** | Physical operator implementation, plus materialization: each logical candidate with every sub-DAG `Lifetime::None` (27: the doc's Raw plans, with tumbling windows rebuilt at every refresh), each tumbling-window chain also `(IngestionTime, AcrossEvaluations(Memory))` (30 more), and each shared input built once per evaluation, `(QueryTime, OneEvaluation(Memory))`, with every tumbling-window chain at query time (15 more) = **72** | | Plan selection | 1 plan | 1 plan | Every MVP candidate is one of the doc's candidates: in logical ASAP-aware @@ -100,26 +100,26 @@ The three whole-window candidates without a shared input: | Id | From | Q2 physical operators | Materialization | |---|---|---|---| -| P1 | L1 | sort (partition by job) → limit 10 | all `NotMaterialized` | -| P10 | L10 | CMS + heap build → top-10 estimate | all `NotMaterialized` | -| P19 | L19 | CountSketch + heap build → top-10 estimate | all `NotMaterialized` | +| P1 | L1 | sort (partition by job) → limit 10 | all `Lifetime::None` | +| P10 | L10 | CMS + heap build → top-10 estimate | all `Lifetime::None` | +| P19 | L19 | CountSketch + heap build → top-10 estimate | all `Lifetime::None` | Q1 is the same in all three: scan → range → rate accumulator → sum accumulator, each finalized. Invariants: -* Every logical candidate keeps its all-`NotMaterialized` physical candidate, +* Every logical candidate keeps its all-`Lifetime::None` physical candidate, and none is pruned, so no candidate is dropped before plan selection. A logical candidate with `n` tumbling-window chains has 2ⁿ physical candidates: each - chain `NotMaterialized` or `MatAtIngestion(Memory)` (ids `P-m`). + chain `Lifetime::None` or `(IngestionTime, AcrossEvaluations(Memory))` (ids `P-m`). Only tumbling-window chains are placed at ingestion time; Q1's rate tumbling windows may be, because continuously ingesting PromQL data is taken to arrive in time order per series. A logical candidate with a shared input has one more: the highest node both queries read (the scan, or the time range above - it) `MatAtQuery(Memory)`, with every chain at query time. A chain at + it) `(QueryTime, OneEvaluation(Memory))`, with every chain at query time. A chain at ingestion time leaves one query reading the input at query time, so - the input stays `NotMaterialized` there. + the input stays `Lifetime::None` there. * Each physical candidate keeps its logical Q2 option and input sharing. * Exact top-*k* is implemented as a sort followed by a limit: each job's series sorted by the ranked value, descending, then a `Project` that @@ -128,7 +128,7 @@ Invariants: returns each job's `k` heaviest series, and a `Project` to `[job, topk_10]` followed by each ranked series' identity. There is no merge node, because there are no window summaries. -* In the whole-window candidates, every node is `NotMaterialized` and runs +* In the whole-window candidates, every node is `Lifetime::None` and runs at query time. ## Plan selection: 1 plan @@ -137,7 +137,7 @@ Invariants: |---|---|---| | P3-m1 | selected | all exact over one time range built once per evaluation for both queries, at 4.38 per second (cost model 1) | | P2-m1 | valid · costlier | all exact over one scan built once per evaluation, at 4.78 per second: each query still reads its own 1 m time range | -| P1, P2, P3 | valid · costlier | all exact at 7.01 per second, with separate inputs (P1) or a `NotMaterialized` shared scan (P2) or scan and range (P3), charged once per query; they tie with each other up to rounding | +| P1, P2, P3 | valid · costlier | all exact at 7.01 per second, with separate inputs (P1) or a `Lifetime::None` shared scan (P2) or scan and range (P3), charged once per query; they tie with each other up to rounding | | P10 | invalid | Count-Min + heap: a top-*k* output has no guarantee (a heap's bound is per item frequency, not top-*k* membership), and the deployment has no evidence that it meets the target | | P19 | invalid | CountSketch + heap: the same reason | @@ -155,7 +155,7 @@ Invariants: reason, and marked invalid or valid but costlier. * Only valid candidates are priced. Each valid candidate's cost has one entry per DAG node, and `cost_per_second` is their sum. A materialized node is charged - once. A `NotMaterialized` node is charged once per consumer that reads it + once. A `Lifetime::None` node is charged once per consumer that reads it (its multiplier), where a consumer is a query or a materialized node and a read stops at materialized nodes. `stage3_plan_selection_charging_helper_*` check the helper on a hand-built candidate with a @@ -185,13 +185,13 @@ per evaluation, within its 100 ms. deployment evidence every heap candidate is invalid and the exact plan is selected. 4. **Physical ASAP-aware optimization is materialization in the doc's - example.** Example 1 lists only window-form and materialization options. + example.** Example 1 lists only window-form and physical properties options. The planner also reads "physical operator implementation" from the physical ASAP-aware optimization section ("TopK as a sort followed by a limit"), and enumerates materialization for summary builds and shared - inputs: each tumbling-window chain is `NotMaterialized` or `MatAtIngestion(Memory)`, - and the highest node both queries read is `NotMaterialized` or - `MatAtQuery(Memory)`. + inputs: each tumbling-window chain is `Lifetime::None` or `(IngestionTime, AcrossEvaluations(Memory))`, + and the highest node both queries read is `Lifetime::None` or + `(QueryTime, OneEvaluation(Memory))`. 5. **Where exact top-*k* becomes sort + limit.** The Pass 1 diagram draws exact Q2 as "sort + limit 10 per job", but the frontend emits `Aggregate{[TopK]}` and the physical ASAP-aware optimization section calls @@ -201,9 +201,9 @@ per evaluation, within its 100 ms. column" without naming the inner sketch. The test assumes `HydraCms`. 7. **"Shared summary charged once"** does not apply literally: the doc says no summary is shared in Example 1. Plan selection charges a - `NotMaterialized` node once per consuming query and a materialized one - once; the tests check that rule. A `NotMaterialized` shared input - therefore saves nothing here; built once per evaluation (`MatAtQuery`), + `Lifetime::None` node once per consuming query and a materialized one + once; the tests check that rule. A `Lifetime::None` shared input + therefore saves nothing here; built once per evaluation (`OneEvaluation`), it is read once, and that candidate (the doc's "Raw with a shared input") is selected. 8. **Latency.** The doc says an exact top 10 rebuilt at every refresh "may diff --git a/docs/design_docs/proposals/planner-layering-example2-acceptance.md b/docs/design_docs/proposals/planner-layering-example2-acceptance.md index cbb2214e7..0ec29bd09 100644 --- a/docs/design_docs/proposals/planner-layering-example2-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example2-acceptance.md @@ -48,8 +48,8 @@ workload itself. | Query-language frontends | One summary-free DAG: `Distinct`, `Entropy`, `L2` over `flows.src_ip`, last 1 m | Same. Q1 is a distinct count; Q2 is `Entropy` and Q3 is `L2` over the per-source counts, each with an exact population count for the empty-input NULL. | | Logical ASAP-aware optimization, Pass 1 | Each computation: exact, a specialized summary, UnivMon. 3 × 3 × 3 = **27** | Q1: exact, HLL, KMV, UnivMon (no Theta: `asap_sketchlib` has no Theta sketch). Q2, Q3: exact, UnivMon (no specialized entropy or norm summary). 4 × 2 × 2 = **16** | | Logical ASAP-aware optimization, Pass 2 | Summary-capability rule: + 1 (one UnivMon for all three, sized for ε = 0.01) + 9 (3 pairs share a UnivMon × 3 options of the third). Independent kept. **37**. Window composition left out. | Identical-expression rule (Q2 and Q3's shared per-source count, and the shared scan) and summary-capability rule (one UnivMon for two of the queries or all three, sized with each parameter's largest value among its readers): **82** | -| Physical ASAP-aware optimization | Same options as Example 1: no window form → rebuilt from raw samples each refresh. | Each logical candidate with every sub-DAG `NotMaterialized`, plus `MatAtQuery(Memory)` for each summary build two or more queries read: **184** | -| Plan selection | One UnivMon sized for ε = 0.01 vs three separate summaries; the shared one usually wins (one update per flow record instead of three). | 164 invalid: HLL or UnivMon for Q1 (120), whose bound has failure probability None without a distinct bound for `flows`, and UnivMon for Q2 (44), whose entropy has no guarantee. 20 valid. Under cost model 1, KMV for Q1 with the other two exact (P33) costs 6.097 per second and P1, all exact, 6.104, the exact distinct count's values held while Q1 runs; the shared per-source count costs 7.46 rebuilt for each query and 7.06 kept `MatAtQuery`; with the shared scan also built once per evaluation, KMV for Q1 (P35-m3) is selected at 5.745, and all exact costs 5.752; UnivMon for Q3 costs 8.09. | +| Physical ASAP-aware optimization | Same options as Example 1: no window form → rebuilt from raw samples each refresh. | Each logical candidate with every sub-DAG `Lifetime::None`, plus `(QueryTime, OneEvaluation(Memory))` for each summary build two or more queries read: **184** | +| Plan selection | One UnivMon sized for ε = 0.01 vs three separate summaries; the shared one usually wins (one update per flow record instead of three). | 164 invalid: HLL or UnivMon for Q1 (120), whose bound has failure probability None without a distinct bound for `flows`, and UnivMon for Q2 (44), whose entropy has no guarantee. 20 valid. Under cost model 1, KMV for Q1 with the other two exact (P33) costs 6.097 per second and P1, all exact, 6.104, the exact distinct count's values held while Q1 runs; the shared per-source count costs 7.46 rebuilt for each query and 7.06 materialized `OneEvaluation`; with the shared scan also built once per evaluation, KMV for Q1 (P35-m3) is selected at 5.745, and all exact costs 5.752; UnivMon for Q3 costs 8.09. | ## Invariants and tests @@ -84,12 +84,12 @@ workload itself. | Invariant | Test | Status | |---|---|---| -| Each logical candidate keeps one physical candidate with nothing materialized, plus one per other materialization assignment; none pruned | `stage2_physical_asap_optimization_keeps_every_logical_candidate` | passes | +| Each logical candidate keeps one physical candidate with nothing materialized, plus one per other physical properties assignment; none pruned | `stage2_physical_asap_optimization_keeps_every_logical_candidate` | passes | | The shared UnivMon stays one build read by all three queries | `stage2_physical_asap_optimization_keeps_the_shared_univmon_as_one_build` | passes | Window-composition candidates are out of scope: the doc leaves them out for this example, and a summary with no window form is rebuilt at every refresh -(Example 1's table) unless a shared build is kept `MatAtQuery` within one +(Example 1's table) unless a shared build is materialized `OneEvaluation` within one evaluation. ### Plan selection @@ -98,7 +98,7 @@ evaluation. |---|---|---| | One selected id; every other candidate rejected once with a reason | `stage3_plan_selection_selects_one_and_explains_the_rest` | passes | | The selected plan is the cheapest valid one | `stage3_plan_selection_selects_cheapest_valid` | passes | -| Each node of a valid candidate is charged once per consuming query while `NotMaterialized`, once when materialized | `stage3_plan_selection_charges_each_node_per_consuming_query` | passes | +| Each node of a valid candidate is charged once per consuming query while `Lifetime::None`, once when materialized | `stage3_plan_selection_charges_each_node_per_consuming_query` | passes | | No candidate builds a Theta sketch | `stage3_plan_selection_has_no_theta_candidate` | passes | | A UnivMon estimate has a guarantee, so it is judged against its target instead of rejected for having none | `stage3_plan_selection_judges_univmon_with_an_accuracy_model` | ignored: needs a UnivMon entropy accuracy model (#788); L2 and the distinct count have one | | Given a distinct bound for the source, classic HLL's failure probability is bound and Q1's HLL is accepted; a larger bound enlarges its precision, also in a tumbling-window chain | `stage3_plan_selection_accepts_hll_given_a_distinct_bound`, `…_enlarges_hll_for_a_larger_distinct_bound`, `…_enlarges_tumbling_window_hll` | passes | @@ -112,8 +112,8 @@ The doc's "the shared candidate usually wins" depends on the cost model, so it is tested only as the dominance above: one build sized for the strictest consumer costs no more than three builds that include one of that size. Whether it also beats the exact plans is the model's call. The dominance needs the -shared UnivMon materialized: plan selection charges a `NotMaterialized` -node once per consuming query, so unmaterialized it costs three builds of the +shared UnivMon materialized: plan selection charges a `Lifetime::None` +node once per consuming query, so not_materialized it costs three builds of the strictest size. ## Ambiguities and conflicts with the planner diff --git a/docs/design_docs/proposals/planner-layering-example3-acceptance.md b/docs/design_docs/proposals/planner-layering-example3-acceptance.md index f6f253b9d..8966edd70 100644 --- a/docs/design_docs/proposals/planner-layering-example3-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example3-acceptance.md @@ -47,7 +47,7 @@ Shared data workload, except Pattern A's `arrival` is `Mixed`. | Query-language frontends | A: five scan → range → quantile chains. B: one. | Same; q3–q5 also have a `time_shift`. | | Logical ASAP-aware optimization, Pass 1 | A: exact or KLL over its own interval per query, 2⁵ = 32. B: exact, KLL. | A: exact, KLL (k = 547), DDSketch per query, 3⁵ = **243**. B: exact, KLL (k = 269), DDSketch, **3**. | | Logical ASAP-aware optimization, Pass 2 | A: window composition adds one EH of KLLs over [T − 5 y, T], one merge + p99 estimate per query, plus a candidate for every grouping of two or more queries onto shared window summaries ("hundreds"). B: each Pass 1 option × {none, sliding L = 5 min s = 1 min (5 active windows), 1-min tumbling (merge 5)} = 6. Independent kept. | Identical-expression rule: each Pass 1 candidate also gets its shared-input candidates (A: 3 per Pass 1 candidate; B: none). Window-composition rule, tumbling windows only: B's KLL and DDSketch each also come as 1-min tumbling windows (a `SummaryTumblingWindowChain` of 5, merged before the p99 estimate), **5** in all; A is run once, so it gets none. No sliding windows or EHs. | -| Physical ASAP-aware optimization | See Example 4. | Each logical candidate with every sub-DAG `NotMaterialized` (A: 972; B: 5), plus, for A, each highest shared input (a scan or a shift) also built once per evaluation, `MatAtQuery(Memory)` (A: 2,187 in all), and, for B, each tumbling-window chain maintained `MatAtIngestion(Memory)` (B: 7 in all). | +| Physical ASAP-aware optimization | See Example 4. | Each logical candidate with every sub-DAG `Lifetime::None` (A: 972; B: 5), plus, for A, each highest shared input (a scan or a shift) also built once per evaluation, `(QueryTime, OneEvaluation(Memory))` (A: 2,187 in all), and, for B, each tumbling-window chain maintained `(IngestionTime, AcrossEvaluations(Memory))` (B: 7 in all). | | Plan selection | Not stated for Example 3. | A: 288 valid, 1,899 invalid: every candidate with a DDSketch misses the rank target (its bound is relative). KLL for all five queries (P485) is selected at 207,189.43 per second (cost model 1); all exact costs 327,132,784.05, about 1,600 times as much, its values held for the whole of each long query. B: KLL (P2, 2.363 per second); the 1-min KLL tumbling windows rebuilt at every evaluation cost 3.611 (the same rows and updates plus merging 5 tumbling windows per series, and the tumbling windows held while the query runs); maintained at ingestion time they keep six tumbling windows of 1,000,000 per-series KLLs for 654.54; exact costs 2.458; DDSketch, whole or in tumbling windows, misses the rank target. Every valid B plan meets 200 ms: 126 ms for the whole-window KLL, 131 ms for the rebuilt tumbling windows, 7 ms for the maintained tumbling windows, 144 ms exact. | ## Invariants and tests @@ -66,7 +66,7 @@ Shared data workload, except Pattern A's `arrival` is `Mixed`. | The EH candidate coexists with the five independent KLLs | `stage1_logical_asap_optimization_a_keeps_independent_and_shared_window_summaries` | ignored: #628, #634 | | Every pair of queries shares a window summary in some candidate | `stage1_logical_asap_optimization_a_window_composition_groups_every_pair` | ignored: #628, #634 | | One selected id, the rest explained; the selected plan is the cheapest valid one | `stage3_plan_selection_a_selects_cheapest_valid` | passes | -| Each node of a valid candidate is charged once per consuming query while `NotMaterialized`, once when materialized | `stage3_plan_selection_a_charges_each_node_per_consuming_query` | passes | +| Each node of a valid candidate is charged once per consuming query while `Lifetime::None`, once when materialized | `stage3_plan_selection_a_charges_each_node_per_consuming_query` | passes | | For the same per-computation choices, sharing the scan costs no more than separate scans | `stage3_plan_selection_a_shared_scan_is_not_costlier` | ignored: #632 (see ambiguity 7) | ### Pattern B diff --git a/docs/design_docs/proposals/planner-layering-example4-acceptance.md b/docs/design_docs/proposals/planner-layering-example4-acceptance.md index ae9371317..3565c9d27 100644 --- a/docs/design_docs/proposals/planner-layering-example4-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example4-acceptance.md @@ -50,12 +50,12 @@ time, both kept. Not materializing it is the no-window plan, so it is not a separate option. **Today:** Pass 2 builds tumbling windows (`SummaryTumblingWindowChain`), and physical -ASAP-aware optimization enumerates `NotMaterialized`, `MatAtIngestion` and -`MatAtQuery` (state reused within one evaluation) for every summary build. +ASAP-aware optimization enumerates `Lifetime::None`, ingestion-time `AcrossEvaluations` and +`OneEvaluation` (state reused within one evaluation) for every summary build. B1 and B2 exist. B3 (tumbling windows built at query time and kept across evaluations) is offered when the deployment declares `query_time_results_retained` in its `deployment_capabilities`: Stage 2 then -adds `MatAtQueryKeptAcrossEvaluations` for a tumbling-window chain that builds +adds query-time `AcrossEvaluations` for a tumbling-window chain that builds one new tumbling window per evaluation. Pattern A's shared EH needs window segments shared across queries (#628, #634), so A1–A3 do not exist yet. B3's 4 kept tumbling windows plus the newest cost 436.59 per second under cost model 1, @@ -63,12 +63,12 @@ between B2 (3.611) and B1 (654.54). ## Invariants and tests -**Reading materialization.** The physical DAG records each node's -`Materialization` (#667). Tests read it through -`planner_layering_common::materialization(p, node)` without the storage -medium (`MatAtIngestion`, `MatAtQuery`, `NotMaterialized`), and the kept +**Reading physical properties.** The physical DAG records each node's +timing and `Lifetime` (#667, #1200). Tests read them through +`planner_layering_common::materializing(p, node)` without the lifetime length or +storage medium (`MaterializedAtIngestion`, `MaterializedAtQueryTime`, `NotMaterialized`), and the materialized event-time span through `retention_ms(p, node)`. The IR does not record -retention, so the adapter derives it the way plan selection prices kept +retention, so the adapter derives it the way plan selection prices materialized state: a chain of `N` tumbling windows of width `w` keeps `N·w`, plus one tumbling window while maintained at ingestion time; a whole-window build keeps its window, plus the next one at ingestion time; each plus the offset of the shifts below @@ -89,7 +89,7 @@ it. | The shared EH yields exactly A1, A2, A3 | `stage2_physical_asap_optimization_a_shared_eh_has_three_materialization_options` | ignored: the shared EH (#628, #634) | | At rest, A2 is not generated; A1 and A3 remain | `stage2_physical_asap_optimization_a_at_rest_drops_the_ingestion_time_option` | ignored: the shared EH (#628, #634) | | A materialized EH is one build node read by all five queries, priced once | `stage2_physical_asap_optimization_a_materialized_eh_is_built_once_for_all_consumers` | ignored: the shared EH (#628, #634) | -| A3 costs strictly more than A1 (five builds instead of one: a `NotMaterialized` node is charged per consuming query) | `stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once` | ignored: the shared EH (#628, #634) | +| A3 costs strictly more than A1 (five builds instead of one: a `Lifetime::None` node is charged per consuming query) | `stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once` | ignored: the shared EH (#628, #634) | | As given, A1 is the cheapest of the three | `stage3_plan_selection_a_once_adhoc_prefers_the_query_time_eh` | ignored: the shared EH (#628, #634) | | Monthly, A2's cost relative to A1 drops: cost(A2)/cost(A1) monthly < as given | `stage3_plan_selection_a_monthly_amortizes_ingestion_time_maintenance` | ignored: the shared EH (#628, #634) | @@ -127,11 +127,11 @@ it. 1. **Charging A3 (decided).** A3 is one logical EH node that is not materialized, so it is rebuilt by each of its five consumers. Plan - selection charges a `NotMaterialized` node once per consuming query + selection charges a `Lifetime::None` node once per consuming query (`NodeCost::multiplier`) and a materialized node once; physical ASAP-aware optimization does not duplicate the node. So A3 costs five EH builds and A1 one, as the tests require. The A3 tests stay ignored only because the - shared EH and its materialization options do not exist yet. + shared EH and its physical properties options do not exist yet. 2. **Recurrence input (resolved).** Materialization depends on `recurrence`, `predictability` and `arrival`: `optimize_physical_asap` takes the workload entries (`QueryWorkloadEntry`: recurrence, diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index cb93a3acb..18ac149bb 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -156,7 +156,7 @@ ASAPPlanner relies on the assumptions below. ### ASAPPlanner System Overview: Planning Stages -ASAPPlanner takes a [query workload](https://github.com/ProjectASAP/ASAPPlanner/blob/main/crates/types/src/workload/mod.rs), a [data workload](https://github.com/ProjectASAP/ASAPPlanner/blob/main/crates/types/src/workload/mod.rs#L384) and the deployment's +ASAPPlanner takes a [query workload](https://github.com/ProjectASAP/ASAPPlanner/blob/main/crates/types/src/workload/query.rs), a [data workload](https://github.com/ProjectASAP/ASAPPlanner/blob/main/crates/types/src/workload/data.rs) and the deployment's inputs (TODO: define this data structure, issue [#525](https://github.com/ProjectASAP/ASAPPlanner/issues/525)), and returns one optimal physical plan. It decides what is computed, how it is computed, and which plan is best. The deployment only supplies data and query inputs and executes the plan: it provides its empirical cost model, empirical accuracy model and capabilities, but never optimize queries. @@ -281,7 +281,7 @@ lists each stage's input and output types. one. It makes no implementation choice, so this section counts it as Stage 0 (its output); the code runs it as Stage 1 starts, which is why [Stage inputs and outputs](#stage-inputs-and-outputs-whole-workload-candidates-and-memo) -lists `generate_dag_of_logical_memo_groups` with Pass 1. `generate_dag_of_logical_memo_groups` runs +lists `memo_of_queries` with Pass 1. `memo_of_queries` runs `share_common_sub_dags` (`crates/logical-asap-optimization/src/stage0/cse.rs`) over the frontends' roots and numbers the shared nodes as groups. Two structurally equal nodes become one group only when sharing them is legal, that is, when every reader @@ -345,7 +345,7 @@ One query, evaluated every minute, with `max_error 0.01` and avg by (job) (quantile_over_time(0.5, data[5m]) / quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m])) ``` -**Step 1, Stage 0.** `generate_dag_of_logical_memo_groups` makes 7 groups: +**Step 1, Stage 0.** `memo_of_queries` makes 7 groups: | Group | Logical sub-expression | Reads | |---|---|---| @@ -506,12 +506,10 @@ into one plan and check and cost it with Stage 2 and Stage 3. The search is the group selection MILP (mixed-integer linear program, #1095): it chooses one candidate per memo group and execution context, with each shareable candidate charged once, and is solved once; it prices cost model 1 only (cost models 2 -and 3 are an `UnsupportedCostModel` error, #888). Stage 3 realizes the chosen -plan through Stage 2's `AssignmentRealizer`, which the facade builds and passes in, -so Stage 3 does not depend on the Stage 2 crate. The facade runs Pass 2, -Stage 2 and Stage 3 once, over the full tumbling-window-width domain, with no -time limit. Stage 3 checks that `plan_cost` of the realized plan -equals the MILP's objective, and that every query's accuracy and latency hold. +and 3 are an `UnsupportedCostModel` error, #888). Stage 3 builds the chosen +plan, a `PhysicalASAPDAG`, as its last step, so it does not depend on the +Stage 2 crate. The facade runs Pass 2, Stage 2 and Stage 3 once, over the +full tumbling-window-width domain, with no time limit. #### Memo code interface @@ -523,12 +521,12 @@ candidates is in `crates/plan-selection/src/` (`select.rs`, `groups/`, `milp/`). | Step | Entry point | Path | |---|---|---| -| Stage 0 | `generate_dag_of_logical_memo_groups` → `DagOfLogicalMemoGroups`, `MemoGroup`, `MemoGroupId` | `stage0.rs` (the types: `asap_types::stages`); hash-consing is `share_common_sub_dags` in `stage0/cse.rs` | -| Pass 1 | `generate_per_computation_logical_memo_candidates` → `PerComputationLogicalMemoCandidates` (its `Memo` and its `PrunedCandidate`s; `ComputationTarget` is Pass 1's working type, in `asap_types::stages`), the one entry: inside it, the computation targets (walking the memo groups), the summaries the deployment does not support and those no query can use pruned, the candidates attached to the groups (`try_populate_pass1`), then the accuracy-scoped sizes | `pass1/logical_memo_candidates.rs`; `pass1/per_computation_candidates.rs`; `pass1/memo_population.rs` | +| Stage 0 | `memo_of_queries` → `DagOfLogicalMemoGroups`, `MemoGroup`, `MemoGroupId` | `stage0.rs` (the types: `asap_types::stages`); hash-consing is `share_common_sub_dags` in `stage0/cse.rs` | +| Pass 1 | `pass1_memo` → `PerComputationLogicalMemoCandidates` (its `Memo` and its `PrunedCandidate`s; `ComputationTarget` is Pass 1's working type, in `asap_types::stages`), the one entry: inside it, the computation targets (walking the memo groups), the summaries the deployment does not support and those no query can use pruned, the candidates attached to the groups (`try_populate_pass1`), then the accuracy-scoped sizes | `pass1.rs`; `pass1/computation_targets.rs`; `pass1/memo_population.rs` | | Memo types | `DagOfLogicalMemoGroups`, `PerComputationLogicalMemoCandidates`, `HowToComputeAggregateMeasure`, Pass 1's working types (`ComputationTarget`, `FusedComputation`, `HavingEntry`, `HavingPlacement`), `Memo`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `CandidateGenerationTimeout`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | -| Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `pass2/logical_memo_candidates.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | +| Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `pass2.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | | Memo to candidate DAGs | `enumerate`, `build_logical_asap_dags_from_chosen_candidates`, `ChosenCandidatePerMemoGroup`, `AllChosenCandidatesPerMemoGroup`, `ErrorBuildingLogicalASAPDagsFromChosenCandidates` | `pass2/memo/enumeration.rs` (re-exported at the crate root) | -| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`generate_per_computation_logical_memo_candidates`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each, and builds the selected plan with Stage 2's `AssignmentRealizer::realize`. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/plan-selection/src/select.rs` | +| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/plan-selection/src/select.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies @@ -567,14 +565,14 @@ The sections below describe each stage. | 0. Frontends | nothing: a query that cannot be lowered fails planning with an error | — | — | | 1, Pass 1 | what the measure or the deployment cannot compute: `RequiresExact`, `NullableAvgInput`, `NoRelationalCountSketchRealization`, `UnsupportedByDeployment`, `QuantileOutsideUnitInterval`, `TopKHeapOverOneRowPerItem`; and, only when the deployment's accuracy model and evidence cannot accept an output without its guarantee, a summary no query reading it can use: `NoErrorBoundForQueryTarget` | `PerComputationLogicalMemoCandidates::pruned_candidates`, each on its aggregate's memo group | the way to compute the measure (`HowToComputeAggregateMeasure`) | | 1, Pass 2 | nothing: it only adds candidates | — | — | -| 2 | what cannot be implemented, built or kept: `PerEntityAggregateNotExecutable` (an exact per-entity `Aggregate` the executor has no kernel for), `RawDataNotRetained` (a query-time scan of raw data the deployment does not keep), `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`, `TooFewQueryReaders` (shared within one evaluation with fewer than two reading queries) | `PhysicalASAPMemoCandidates::pruned_candidates`, each on its operator node's memo group | `PhysicalCandidateKey { operator_node, timing, materialization }` | +| 2 | what cannot be built or materialized: `NoIngestionPipeline`, `RawDataNotRetained` (a query-time scan of raw data the deployment does not keep), `QueryTimeResultsNotRetained`, `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`; the planner does not check executor support | `PhysicalASAPMemoCandidates::pruned_candidates` | the `PhysicalCandidate` it would have been | | 3 | valid candidates that lose, and the checks only a whole plan can make (accuracy, latency, cluster capacity) | its own `RejectionCause` | — | Stage 2's key names what a pruned candidate would have been, since kept candidates are renumbered after pruning and a pruned one has no `PhysicalCandidateId`. Stage 2 emits one candidate per operator node, -timing and materialization, and makes no copies of it, so (`operator_node`, -`timing`, `materialization`) is unique. Operator +timing and lifetime, and makes no copies of it, so (`operator_node`, +`properties`) is unique. Operator nodes are kept whole, so `operator_node` indexes the output's `operator_nodes_of_candidates_for_memo_groups`. Pass 1's accuracy pruning (`NoErrorBoundForQueryTarget`) runs inside Pass 1's entry, before the @@ -597,10 +595,10 @@ the memo path's entry points. | Stage | Whole-workload candidates: input → output | Memo: input → output | Entry point (memo path) | |---|---|---|---| | 0. Frontends | Query text, language → one logical root per query, carrying its accuracy target | Same: a `QueryRoot` per query | `lower_promql_query_workload`, `lower_sql_dialect` (`asap-frontend-*`); `plan_installable` (`asap_planner`) runs the frontends and Stages 1–3 on the user input | -| 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (`PerComputationCandidates`, now Pass 1's crate-private working form) | Logical roots → `DagOfLogicalMemoGroups` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support and those no query reading them can use → `PerComputationLogicalMemoCandidates`: the `Memo` (each group's `CandidateForMemoGroup`s, with the accuracy-scoped sizes) and the pruned candidates (`PrunedCandidate`) | `generate_dag_of_logical_memo_groups`, `generate_per_computation_logical_memo_candidates` | +| 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (`PerComputationCandidates`, now Pass 1's crate-private working form) | Logical roots → `DagOfLogicalMemoGroups` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support and those no query reading them can use → `PerComputationLogicalMemoCandidates`: the `Memo` (each group's `CandidateForMemoGroup`s, with the accuracy-scoped sizes) and the pruned candidates (`PrunedCandidate`) | `memo_of_queries`, `pass1_memo` | | 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → the same memo with each rule's candidates added to the groups they implement | the memo rules (`pass2/memo/rules/`), `pass2::memo::window_candidates` | | 1, concrete summary-state groups | (part of Pass 2) | `Memo` → the memo with concrete summary-state groups: one group per shared size or tumbling-window layout that readers can read | `add_shared_summary_state_groups`, Pass 2's last step; the Pass 2 output is `LogicalASAPMemoCandidates` (`generate_logical_asap_memo_candidates`) | -| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per materialization assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and materialization (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_physical_asap_optimization`) | +| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_physical_asap_optimization`) | | 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | | 4. Execution | The selected `PhysicalASAPDAG` → query results | Same: an `InstallablePlan` (`{dag, query_texts}`) → query results | The deployment runtime installs the `InstallablePlan` | @@ -611,7 +609,7 @@ What changed with the memo: jointly. - Stage 2 is no longer a pass over whole candidate DAGs. It implements every operator node of every candidate for a memo group once, enumerates each - node's timings and materializations, and prunes what cannot be built, + node's timings and lifetimes, and prunes what cannot be built, used or kept; Stage 3 selects among what remains. - The concrete summary-state groups are expanded by Stage 3's search today, through the Stage 1 functions above. A planned change moves this expansion @@ -844,7 +842,7 @@ tumbling windows keeps the guarantee when the family's merged error is `Exact` o `AsIfBuiltOverUnion`), and a build shared by several readers takes the strictest reader's size, one state group per distinct size. The failure probability is part of Pass 1's size too -(`generate_per_computation_logical_memo_candidates`): each approximate input +(`pass1_memo`): each approximate input of a candidate with `n` budget parts is sized for `δ / (n · V)`, the smallest over the queries reaching its group, where `V` is the number of values the query's accuracy scope covers (1 per value, at most `⌊1 / share⌋` for a record @@ -1120,9 +1118,9 @@ kept and more tumbling windows merged, so Stage 3's pricing decides which grid, any, pays; no tumbling-window-count limit is imposed beyond the tumbling-window count fitting in 32 bits. The chain covers the longest window and is sized for all of them, each parameter at its largest value among them, as the -summary-capability rule sizes a shared build. Each query merges only the -newest tumbling windows covering its own window (`SummaryMerge.newest_tumbling_windows`), since a -shorter window ending at the same time is the newest part of the longest. +summary-capability rule sizes a shared build. Each query merges the +windows a `TimeRange` over the chain selects for its own range; the chain +has no range and no window count of its own. For example, `quantile_over_time(0.5, m[1m])` and `quantile_over_time(0.9, m[5m])`, both every 10 s, read one chain of thirty 10-s KLL tumbling windows: the 1-min query merges the newest 6, the 5-min query all @@ -1198,21 +1196,26 @@ the `DataWorkload`. Typical outcomes: A shared summary is materialized once for all its consumers. See Example 4. -The planner implements a subset of these options today. Stage 2 enumerates +The planner implements a subset of these options today. In code each option +is a timing and a lifetime (`PhysicalProperties`, #1200): materialized at +ingestion time is ingestion time with lifetime `AcrossEvaluations`; +materialized at query time is query time with lifetime `OneEvaluation` +(reused within one evaluation) or `AcrossEvaluations` (materialized for later +evaluations); not materialized is lifetime `None`. Stage 2 enumerates materialization for summary builds (`SummaryAgg` and `SummaryTumblingWindowChain`); each -gets `NotMaterialized`, `MatAtIngestion(Memory)` when it is eligible to run -at ingestion time, and `MatAtQuery(Memory)` when two or more queries read -it. Of every other sub-DAG that two or more queries read (for example a -`Scan` that a 1 m and a 5 m window both read), only the highest node is a -materialization candidate: one that a query's output, or a node only one -query reads, consumes. It gets `NotMaterialized` and `MatAtQuery(Memory)` -only, since raw rows are not maintained at ingestion time. Below an -ingestion-time build it stays `NotMaterialized`: the ingestion side reads -its own copy. A `MatAtQuery` state is reused within one evaluation only. When the -deployment's capabilities declare `query_time_results_retained`, a -tumbling-window chain that builds exactly one new tumbling window per evaluation also gets -`MatAtQueryKeptAcrossEvaluations(Memory)`: the tumbling windows built at query time are -kept for the next evaluations. Nothing is placed on disk yet. +is offered not materialized, materialized at ingestion time `AcrossEvaluations(Memory)` when +it is eligible to run at ingestion time, and materialized at query time +`OneEvaluation(Memory)` when two or more queries read it. Of every other +sub-DAG that two or more queries read (for example a `Scan` that a 1 m and a +5 m window both read), only the highest node is a materialized-output candidate: +one that a query's output, or a node only one query reads, consumes. It is +offered not materialized and `OneEvaluation(Memory)` only, since raw rows are not +maintained at ingestion time. Below an ingestion-time build it stays not +materialized: the ingestion side reads its own copy. When the deployment's +capabilities declare `query_time_results_retained`, a tumbling-window chain +that builds exactly one new tumbling window per evaluation is also offered +query time `AcrossEvaluations(Memory)`: the tumbling windows built at query +time are materialized for the next evaluations. Nothing is placed on disk yet. A runtime builds every ingestion-time state on the grid of its width anchored at the epoch, so an ingestion-time build must end on that grid at every @@ -1243,20 +1246,20 @@ Output: C3 — CandidatePhysicalASAPDAGs 2: for each LogicalASAPDAG d in C2 do 3: // Materialization: options per sub-DAG 4: for each sub-DAG s in d do -5: Opt(s) ← { NotMaterialized } +5: Opt(s) ← { (QueryTime, None) } 6: for each medium in {memory, disk} do -7: Opt(s) ← Opt(s) ∪ { MatAtIngestion(medium), MatAtQuery(medium) } +7: Opt(s) ← Opt(s) ∪ { (IngestionTime, AcrossEvaluations(medium)), (QueryTime, OneEvaluation(medium)) } 8: end for 9: end for 10: 11: for each assignment m in Opt(s_1) × … × Opt(s_n) do 12: // constraint: everything upstream of an ingestion-time node also runs at ingestion time -13: if some s with m(s) = MatAtIngestion has an upstream node u with m(u) ≠ MatAtIngestion then +13: if some s with timing(m(s)) = IngestionTime has an upstream node u with timing(m(u)) ≠ IngestionTime then 14: PRUNE(m, "upstream of ingestion-time node not at ingestion time") 15: continue 16: end if 17: // constraint: keep a materialized output as long as any consumer needs it -18: for each s with m(s) ≠ NotMaterialized do +18: for each s with lifetime(m(s)) ≠ None do 19: retention(s) ← latest time any consumer of s reads it // from recurrence, windows 20: end for 21: // a shared summary is one node, so it is materialized once for all consumers @@ -1281,15 +1284,13 @@ cheapest plan (the typical outcomes above). On the memo, Stage 2 prunes per operator node and records each pruned candidate in `PhysicalASAPMemoCandidates::pruned_candidates` ([Pruning by stage](#pruning-by-stage)). It does not generate a -configuration the deployment cannot run: an exact per-entity `Aggregate` -(a per-series range reduction over raw data samples) with more than one -measure, a measure without an exact per-entity kernel such as -`stddev_over_time`, a measure filter or a `HAVING` -(`PerEntityAggregateNotExecutable`; the rule reads one node, so no whole -plan is needed), and, without raw data retained, a query-time scan of raw -data (`RawDataNotRetained`). It then removes the candidates no buildable -input, query output or reader count can use (`InputHasNoBuildableProducer`, -`ReachesNoQueryOutput`, `TooFewQueryReaders`), +configuration the deployment cannot run: an ingestion-time alternative +without an ingestion pipeline (`NoIngestionPipeline`), a query-time scan +of raw data the deployment does not retain (`RawDataNotRetained`), and a +materialized query-time result the deployment does not retain +(`QueryTimeResultsNotRetained`). It does not check executor support. It +then removes the candidates no buildable input or query output can use +(`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`), repeating until nothing more is removed. Stage 2 makes no execution-scope copies: how many times a candidate runs is Stage 3's choice, and a selection names each run as a node of its `DagOfSelectedCandidates`. @@ -1299,7 +1300,7 @@ into a node. *Example for Algorithm 3* ([Example 4, Pattern B](#example-4-materialization-of-window-summaries-in-physical-planning)). The logical candidate is the `Tumbling(1m)` KLL option from Algorithm 2. The -diagram shows three of its materialization assignments, each drawn as a copy +diagram shows three of its physical properties assignments, each drawn as a copy of the DAG with every node colored by its materialization. Line 13 enforces one rule: a node can run at ingestion time only if every node it reads from also runs at ingestion time, because otherwise its input does not exist yet when @@ -1328,7 +1329,7 @@ flowchart TB end subgraph LEG["Legend: when a node runs and whether its output is stored"] direction LR - l1["MatAtIngestion"]:::ingest ~~~ l2["MatAtQuery"]:::qtime ~~~ l3["NotMaterialized"]:::notmat + l1["(IngestionTime, AcrossEvaluations)"]:::ingest ~~~ l2["(QueryTime, OneEvaluation)"]:::qtime ~~~ l3["(QueryTime, None)"]:::notmat end M1 --> P1 LEG ~~~ M1 diff --git a/docs/design_docs/proposals/window-materialization-choices.md b/docs/design_docs/proposals/window-materialization-choices.md index 7646956cd..026edabfb 100644 --- a/docs/design_docs/proposals/window-materialization-choices.md +++ b/docs/design_docs/proposals/window-materialization-choices.md @@ -22,5 +22,5 @@ A tumbling window is an aligned, non-overlapping fixed-width interval and its ag This table describes possible strategies. Current tumbling window support does not imply general direct sliding-state or EH support. Offer only executable candidates. Multiple tumbling-window widths can coexist; alignment, coverage, mergeability, retention, and error contracts determine legality. A 15m window evaluated every 1m can use 1m tumbling windows; complete 5m tumbling windows alone generally cannot cover every endpoint. -Use existing `Materialization` options: `NotMaterialized`, `MatAtQuery(Memory)`, `MatAtIngestion(Memory)`, and `MatAtQueryKeptAcrossEvaluations(Memory)`. Assign them to eligible internal nodes, not necessarily the entire query. For example, maintain tumbling windows at ingestion and merge at query time. Generate only supported assignments: retained query-time state currently requires deployment capability and an eligible tumbling-window chain. Ingestion timing and upstream dependencies must satisfy Stage 2 rules. +Use existing physical properties: `Lifetime::None`, `(QueryTime, OneEvaluation(Memory))`, `(IngestionTime, AcrossEvaluations(Memory))`, and `(QueryTime, AcrossEvaluations(Memory))`. Assign them to eligible internal nodes, not necessarily the entire query. For example, maintain tumbling windows at ingestion and merge at query time. Generate only supported assignments: retained query-time state currently requires deployment capability and an eligible tumbling-window chain. Ingestion timing and upstream dependencies must satisfy Stage 2 rules. diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index d638c7a36..f22f1d0d5 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -17,10 +17,10 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with | Crate | Role | | --- | --- | -| `asap-types` (`crates/types`) | Shared vocabulary, by IR section: `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator parameters, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`, summary-state types such as `SketchAlgorithm` and `FamilyCapability`), `ir::properties` (`AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`, execution timing, summary coverage), plus `ir::flat` and `ir::physical_export`; the planner inputs (`workload`); Stage 2 decision data (`physical`: `Materialization`, tumbling windows) | +| `asap-types` (`crates/types`) | Shared vocabulary, by scope: the operator IR by IR section, `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator arguments, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`), `ir::properties` (execution timing, summary coverage) and `ir::flat`; summary types (`summary_type`: `SketchAlgorithm`, `SketchParams`, `SummaryTypeCapability`); accuracy targets and guarantees (`accuracy`: `AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`); the planner inputs (`workload`); the stage-boundary types (`stages`, with Stage 2's `PhysicalProperties` (timing and `Lifetime`)); and Stage 3's plan (`plan`: `plan::physical_asap_dag`, `plan::materialized_view`) | | `asap-frontend-sql`, `asap-frontend-promql`, `asap-frontend-metricsql` (with `asap-frontend-common`) | Stage 0, the query-language frontends: lower each query into the logical DAG (`QueryRoot`s of `NonASAPOp` nodes, no summaries, no timing) | | `asap-logical-asap-optimization` (`crates/logical-asap-optimization`) | Stage 1: Pass 1 and Pass 2 (the memo rules) | -| `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-family estimators, composition, budget split, target check) and the deployment's `AccuracyModel` and `AccuracyEvidenceProvider`; depends only on `asap-types` | +| `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-algorithm estimators, composition, budget split, target check), the deployment's measured accuracy (`AccuracyEvidence`) and α^log (`producer_satisfies_logical_requirements`); depends only on `asap-types` | | `asap-physical-asap-optimization` (`crates/physical-asap-optimization`) | Stage 2: materialization and physical operator implementation | | `asap-cost` (`crates/cost`) | The cost model the stages share: resource-usage estimation (the analytical per-operator model, measured operation costs, state byte sizes) and the cost models (`ResourceUsage`, `CostModel`, `CostInputs`); depends only on `asap-types` | | `asap-plan-selection` (`crates/plan-selection`) | Stage 3: `select_plan` over Stage 2's output; accuracy and latency checks, pricing candidates with `asap-cost` | @@ -42,8 +42,8 @@ asap-types ◄── asap-executor (Stage 4; no stage crate depends on it) ```text QueryRoots (Stage 0) - │ generate_dag_of_logical_memo_groups → DagOfLogicalMemoGroups, then - │ generate_per_computation_logical_memo_candidates + │ memo_of_queries → DagOfLogicalMemoGroups, then + │ pass1_memo ▼ Stage 1, Pass 1 PerComputationLogicalMemoCandidates (Memo + pruned_candidates) │ generate_logical_asap_memo_candidates @@ -53,31 +53,31 @@ LogicalASAPMemoCandidates │ asap_physical_asap_optimization::generate_physical_asap_memo_candidates Stage 2 ▼ PhysicalASAPMemoCandidates (operator nodes per timing/materialization + pruned_candidates) - │ asap_plan_selection::select_plan (with Stage 2's AssignmentRealizer) Stage 3 + │ asap_plan_selection::select_plan Stage 3 ▼ -Selection (selected_dag, cost, accepted_accuracy_per_query, search_report) +(PhysicalASAPDAG, Selection) — the plan, and accepted_accuracy_per_query, +pruned_candidates and the selection MILP's size and time ``` `asap_planner::run_planning` runs all of it over lowered roots: Pass 1, then -Pass 2 over every tumbling-window width, Stage 2 and Stage 3, and builds the -selected plan with `AssignmentRealizer::realize`. +Pass 2, Stage 2 and Stage 3, which builds the selected plan. It returns every stage's output (`PlanningRun`); `Stage1ToStage3Pass` is the same pipeline behind the `OptimizationPass` trait. | Concern | Where it lives | | --- | --- | -| Pass 1: which computations a query has (`ComputationTarget`), how each can be computed (`HowToComputeAggregateMeasure`: `Exact`, `ExactAccumulator`, `Summary`), fused computations (`FusedComputation`), pruning (`PruneReason`) and composing one choice into a logical DAG (`compose_logical_candidate`) | `asap-logical-asap-optimization`, `pass1/per_computation_candidates.rs` | +| Pass 1: which computations a query has (`ComputationTarget`), how each can be computed (`HowToComputeAggregateMeasure`: `Exact`, `ExactAccumulator`, `Summary`), fused computations (`FusedComputation`), pruning (`PruneReason`) | `asap-logical-asap-optimization`, `pass1/computation_targets.rs` | | Which summary families serve an intent (`summary_candidates`) | `asap-logical-asap-optimization`, `pass1/realization.rs` | | Pass 2 memo rules: shared producers (identical expressions by memo group identity, one build at different sizes), scan-label widening, window composition (tumbling windows), rollup, exact top-k reuse and finer-aggregate reuse | `asap-logical-asap-optimization`, `pass2/memo/rules/` and `pass2/memo/window_candidates.rs`, over the helpers in `pass2/asap_aware_cse.rs` | | Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | `asap-logical-asap-optimization`, `pass2/memo/enumeration.rs` | -| What each summary family can merge and answer (`FamilyCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `ir/schema/family_capability.rs` | -| Stage 2: materialization options per summary build (`NotMaterialized`, `MatAtIngestion`, `MatAtQuery`, `MatAtQueryKeptAcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-physical-asap-optimization` | -| Timing and data-state edge rules applied from a `MaterializationAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `Materialization` in `physical/materialization.rs` | +| What each summary type can merge and answer (`SummaryTypeCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `summary_type/capability.rs` | +| Stage 2: physical properties options per summary build (not materialized; materialized at ingestion time `AcrossEvaluations`; materialized at query time `OneEvaluation` or `AcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-physical-asap-optimization` | +| Timing and data-state edge rules applied from a `PhysicalPropertiesAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `PhysicalProperties` and `Lifetime` in `physical/properties.rs` | | Stage 3: accuracy check, latency check, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-plan-selection`, `lib.rs`; `asap-cost`, `lib.rs` and `measured.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `analytical.rs` and `operator_statistics.rs` | -| The accuracy algebra: per-family guarantees, composition, budget split; guarantee propagation through a DAG; sizing a summary for its target, (ε, δ) budget and default sizes | `asap-accuracy` (`AccuracyModel`, `BuiltinAccuracyModel`, `estimators/`, `composition.rs`, `budget.rs`, `propagation.rs`, `sizing.rs`) | +| The accuracy algebra: per-family guarantees, composition, budget split; guarantee propagation through a DAG; sizing a summary for its target, (ε, δ) budget and default sizes | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `requirements.rs`, `error_propagation.rs`, `composition.rs`, `budget.rs`, `propagation.rs`, `sizing.rs`) | | What the executor supports (`DeploymentCapabilities`) | `asap-logical-asap-optimization`, `deployment_capabilities.rs`; Pass 1 receives only `supported_summaries` and Stage 2 only `query_time_results_retained` and `raw_data_retained` (`bool` arguments, so Stage 2 does not depend on the Stage 1 crate); Stage 3 reads none of it | -| The inputs every stage reads (`PlanningModels`: accuracy model, evidence, cost inputs, deployment capabilities) and the `OptimizationPass` trait | `asap-planner`, `pass.rs`; the facade gives Stage 3 only `AccuracyCheckInputs` and `CostInputs` | +| The inputs every stage reads (`PlanningModels`: accuracy evidence, cost inputs, deployment capabilities) and the `OptimizationPass` trait | `asap-planner`, `pass.rs`; the facade gives Pass 2 the evidence and Stage 3 only `CostInputs` | The current production search organizes these computations in memo result/state groups and jointly selects physical candidates and connections; it does not @@ -98,8 +98,8 @@ The unwired maintained-population rule has been removed as well; no Stage 1 pass For each composed DAG, the Pass 2 rules find what can be shared, split into tumbling windows or reused, and each subset of their candidates becomes another Stage 1 candidate. -3. **Stage 2** gives each summary build its materialization options and emits - one physical candidate per assignment (the all-`NotMaterialized` one +3. **Stage 2** gives each summary build its physical properties options and emits + one physical candidate per assignment (the all-not-materialized one first), skipping assignments that put a build at ingestion time over one that is not. 4. **Stage 3** checks each physical candidate: every query's root guarantee @@ -121,7 +121,7 @@ No stage caps the candidates it plans, and neither does `asap_planner`. Pass 1. - `crates/integration-tests/tests/planner_layering_example{1,2,3,4}.rs` pin the design document's examples through every stage; - `reuse_rules.rs`, `deployment_capabilities.rs`, `materialization_e2e.rs` + `reuse_rules.rs`, `deployment_capabilities.rs`, `physical_properties_e2e.rs` and `stage1_logical_asap_optimization.rs` cover single rules end to end. - `crates/devtools/tests/asap_planner.rs` checks the stage document and the committed viewer examples; `asap_planner_acceptance.rs` checks diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 6086507c2..4935b3c26 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -14,13 +14,13 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types | After | The DAG holds | Timing | | --- | --- | --- | | Stage 0 (frontends) | `NonASAPOp` nodes only; aggregates carry `AggIntent` measures with each query's `AccuracyTarget` as written | none | -| Stage 1 (`compose_logical_candidate`, Pass 2 rules) | `ASAPOp` nodes where a candidate chose an exact accumulator or a summary (`SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge`, `SummaryEstimate`, `FinalizeExactAccumulator`); everything else stays `NonASAPOp` | none: Stage 1 never decides when anything runs | -| Stage 2 (`generate_physical_asap_memo_candidates`; a selected plan built by `Stage2AssignmentRealizer`) | the same operators, with a `Materialization` on each summary build | every node timed from the candidate's `MaterializationAssignment` | +| Stage 1 (Pass 1 and Pass 2 rules) | `ASAPOp` nodes where a candidate chose an exact accumulator or a summary (`SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge`, `SummaryEstimate`, `FinalizeExactAccumulator`); everything else stays `NonASAPOp` | none: Stage 1 never decides when anything runs | +| Stage 2 (`generate_physical_asap_memo_candidates`) and Stage 3 (`select_plan`, which builds the plan) | the same operators, with a timing and a `Lifetime` on each summary build | every node of the plan timed from the physical candidate it was built from (`selected_physical_candidate`) | - Structurally equal sub-DAGs are interned before Stage 1 (`stage0/cse.rs` in `asap_logical_asap_optimization`), so Stage 1 compares nodes by structure and Pass 2 matches readers structurally, never by pointer. -- Timing is applied with `apply_materialization_timings`, which validates +- Timing is applied with `apply_physical_properties`, which validates every data-state edge. A violation is an `ExecutionDataStateError`: for example `QueryTimeValueUnderIngestionSummary` (a query-time value beneath an ingestion-time build), `ConflictingTiming` (one shared node @@ -62,11 +62,12 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types ## 3. Accuracy -- `AccuracyModel` is the extension point: `local_guarantee` (a family's - guarantee over exact input), `propagate` (composition through an operator) - and `satisfies` (a guarantee against a target). It fails closed: no rule, or - a missing input guarantee, means no guarantee, and the candidate is - rejected rather than assumed accurate. +- A summary's guarantee is the deployment's measurement + (`AccuracyEvidence::measured_guarantee`) when there is one, else its + algorithm's error formula (`theoretical_guarantee`); Pass 2 compares it + with each readout's `RequiredAccuracy` + (`producer_satisfies_logical_requirements`). It fails closed: no rule + means no guarantee, and the edge is not admissible. - A target with no `accuracy_metric` accepts a guarantee in any metric that meets its ε, δ and scope; Stage 3 reports the metric it accepted (`AcceptedAccuracy`). @@ -78,27 +79,29 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types ## 4. Stage 2 - Only summary builds (`SummaryAgg`, `SummaryTumblingWindowChain`) get materialization - options: `NotMaterialized` always; `MatAtIngestion(Memory)` when the build - is eligible to run as data arrives; `MatAtQuery(Memory)` when two or more - queries read it (reused within one evaluation); - `MatAtQueryKeptAcrossEvaluations(Memory)` for a tumbling-window chain that + options, each a timing and a lifetime (`PhysicalProperties`): not materialized always; + materialized at ingestion time, `AcrossEvaluations(Memory)`, when the build is + eligible to run as data arrives; materialized at query time for `OneEvaluation(Memory)` + when two or more queries read it (reused within one evaluation); materialized at + query time `AcrossEvaluations(Memory)` for a tumbling-window chain that builds one new tumbling window per evaluation, when the deployment declares `query_time_results_retained`. -- One `PhysicalCandidate` per assignment, the all-`NotMaterialized` one first +- One `PhysicalCandidate` per assignment, the all-not-materialized one first (`P`, then `P-m`). An assignment that puts a build at ingestion time over one that is not is not generated. A logical candidate Stage 2 cannot implement becomes a `PrunedCandidate` with its reason. - On the memo (`generate_physical_asap_memo_candidates`), every physical candidate Stage 2 prunes is recorded in `PhysicalASAPMemoCandidates::pruned_candidates` as - `PrunedCandidate` with a `PruneReason`: an exact - per-entity `Aggregate` the executor cannot run - (`PerEntityAggregateNotExecutable`) and, without raw data retained, a - query-time scan of raw data (`RawDataNotRetained`) are not generated; a - candidate no buildable input, query output or reader count can use is - removed (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`, - `TooFewQueryReaders`). Stage 2 emits one candidate per operator node, - timing and materialization; it makes no execution-scope copies. + `PrunedCandidate` with a `PruneReason`: an + ingestion-time alternative without an ingestion pipeline + (`NoIngestionPipeline`), a query-time scan of raw data the deployment does + not retain (`RawDataNotRetained`) and a materialized query-time result the + deployment does not retain (`QueryTimeResultsNotRetained`) are not + generated; a candidate no buildable input or query output can use is + removed (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`). The + planner does not check executor support. Stage 2 emits one candidate per operator node, + timing and lifetime; it makes no execution-scope copies. ## 5. Stage 3 @@ -131,5 +134,5 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types | What is shared, split into tumbling windows or reused | Pass 2 | | When each build runs and whether it is kept | Stage 2 | | Which plan is valid and which is chosen | Stage 3 | -| What the executor can build, retain and how it is priced | the deployment, through `PlanningModels` (`DeploymentCapabilities`, `CostInputs`, `AccuracyModel`) | +| What the executor can build, retain and how it is priced | the deployment, through `PlanningModels` (`DeploymentCapabilities`, `CostInputs`, `AccuracyEvidence`) | | Running the plan | the executor (`asap-executor`), outside the planner | diff --git a/docs/develop_docs/end-to-end-accuracy-guarantees.md b/docs/develop_docs/end-to-end-accuracy-guarantees.md index 4374708cf..43aa9641b 100644 --- a/docs/develop_docs/end-to-end-accuracy-guarantees.md +++ b/docs/develop_docs/end-to-end-accuracy-guarantees.md @@ -39,7 +39,7 @@ The main implementation locations are: | Concern | Location | | --- | --- | -| Guarantee and error vocabulary | `asap_types::ir::properties::guarantee` | +| Guarantee and error vocabulary | `asap_types::accuracy` | | Accuracy model and built-in propagation | `asap_accuracy` | | Candidate construction and legality filtering | `asap_aware_mapping::replacement` | | Parameter sizing hooks | `asap_aware_mapping::cost_model` | @@ -52,23 +52,26 @@ Read the sections below when changing one of those contracts. Accuracy reasoning and allocation are separate from cost modeling: ```rust -trait AccuracyModel { - fn local_guarantee(/* family, query, parameters */) - -> Option; - - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, - ) -> Result; - - fn satisfies( +/// The theoretical error bound of reading `statistic` from a state of +/// `summary_type` with parameters θ at `failure_probability`: the +/// algorithm's error formula. `None` when θ cannot fail that rarely. +pub fn theoretical_guarantee( + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, + failure_probability: f64, +) -> Option; + +/// The deployment's measured error bound for the same configuration, read +/// first when present; `NoMeasuredGuarantees` has none. +pub trait AccuracyEvidence { + fn measured_guarantee( &self, - guarantee: &AccuracyGuarantee, - target: &AccuracyTarget, - ) -> bool; + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, + failure_probability: f64, + ) -> Option; } ``` @@ -174,9 +177,7 @@ against the DataSketches maximum-rank-error characterization `2.446 / k^0.9433` at a per-trial failure probability of 0.01, not against this single-sided fit or its δ scaling. -A KLL shared by readers with different failure probabilities is sized at the -smallest one. Each reader's `k` is first resized to it at no larger rank -error (`kll::shared_k`), and the largest is kept. See +See [Apache DataSketches KLL accuracy](https://datasketches.apache.org/docs/KLL/KLLAccuracyAndSize.html) and its [C++ contract pinned at `a9b42755072b`](https://github.com/apache/datasketches-cpp/blob/a9b42755072b079fd90b29b9851adc121015c58e/kll/include/kll_sketch.hpp). @@ -204,7 +205,7 @@ satisfy a target with only `error` set, but not one that also sets `failure_probability`. Classic HLL (its linear-counting branch) has an estimator-specific confidence -bound (`accuracy/estimators/hll.rs`). Its precision is sized from (ε, δ), and +bound (`accuracy/guarantee/hll.rs`). Its precision is sized from (ε, δ), and its failure probability stays symbolic, `classic_hll_failure_probability[p=P,eps=ε](max_distinct[source])`, until plan selection binds it with the source's @@ -259,9 +260,9 @@ with probability at most `1/3`. Zero or even depth has no modeled guarantee. ASAPPlanner contains the guarantee algebra and parameter-derived contracts. Planner is not a query executor. Data- or runtime-dependent evidence enters -through an `AccuracyEvidenceProvider`, is exposed to propagation as typed -`PropagationStats`, and is recorded in provenance. `NoAccuracyEvidence` is the -default: it supplies no missing facts. Guarantee derivation remains conservative. +through `AccuracyEvidence` (measured guarantees) or the data workload's +declared facts, is exposed to propagation as typed `PropagationStats`, and +is recorded in provenance. The default evidence has no measurements. Guarantee derivation remains conservative. ### TopK membership diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index 9c03a415e..a95a8d4fe 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -10,15 +10,16 @@ owns the decision. A new sketch that Pass 1 can offer for an intent. -1. **Type.** `SketchAlgorithm` is `asap_sketchlib`'s family enum, so the - family must exist in the library first. Add its parameters to - `SketchParams` (`crates/types/src/ir/schema/sketch.rs`); a Hydra kind - maps to its library family in `FamilyCapability::of`. +1. **Type.** `SketchAlgorithm` is `asap_sketchlib`'s algorithm enum, so the + algorithm must exist in the library first; its summary type is + `FieldDataType::Sketch(algorithm)`. Add its parameters to `SketchParams` + (`crates/types/src/summary_type/parameters.rs`) and its predefined list to + `parameter_list`. 2. **Capability.** Its row, which inputs it can merge and which statistics it answers with the merged error, is the library's family-capability table (`SketchAlgorithm::capability`), read unchanged by - `FamilyCapability::of_per_instance_sketch` - (`crates/types/src/ir/schema/family_capability.rs`). Pass 2's sharing, + `SummaryTypeCapability::of` + (`crates/types/src/summary_type/capability.rs`). Pass 2's sharing, tumbling window and rollup rules read only this table. Which readouts the executor evaluates is its own declaration, `EVALUATED_READOUTS` (`crates/executor/src/capability.rs`); add the family's readouts there @@ -28,14 +29,14 @@ A new sketch that Pass 1 can offer for an intent. that order, for every intent whose statistic its capability row answers. Size it in `asap_accuracy::sizing::default_size_params`. 4. **Accuracy.** Add its error model under - `crates/accuracy/src/estimators/` and return - it from `BuiltinAccuracyModel`'s `local_guarantee`, by inverting the same - sizing formulas. Without a guarantee, every candidate that uses it is + `crates/accuracy/src/guarantee/` and return + it from `theoretical_guarantee`, evaluated at the required failure + probability. Without a guarantee, every candidate that uses it is rejected. 5. **Cost.** Give it a state size (the library's `estimated_memory_bytes` for the type the executor builds; KLL keeps the planner's own formula) and per-operation CPU in `sketch_shape` - (`crates/cost/src/state_bytes.rs`), and accept its parameters in + (`crates/cost/src/node/state_bytes.rs`), and accept its parameters in `crates/cost/src/measured.rs` if deployments measure it. 6. **Executor.** Wrap the `asap_sketchlib` implementation in `crates/executor/src/summary_kernels/` and register it in @@ -58,10 +59,10 @@ work. (for example `rollup.rs` or `topk.rs`): it adds candidates (and, when it needs them, groups) to the memo groups it implements, keeping every existing candidate. Read summary families only through - `FamilyCapability`. Shared rewrite helpers live in + `SummaryTypeCapability`. Shared rewrite helpers live in `pass2/asap_aware_cse.rs`. 2. Declare it in `pass2/memo/rules.rs` and call it from `apply_sharing_rules` - (`pass2/logical_memo_candidates.rs`), after the + (`pass2.rs`), after the existing rules. 3. Add its row to the Pass 2 rule table of the [design](../design_docs/proposals/planner-layering.md). @@ -74,20 +75,21 @@ and when it must not; an end-to-end test like `crates/integration-tests/tests/reuse_rules.rs` that the planned workload contains the candidate and that Stage 3 prices it. -## 3. Add a Stage 2 materialization option +## 3. Add a Stage 2 physical properties option -1. Add the `Materialization` variant - (`crates/types/src/physical/materialization.rs`) and its timing in +1. Add the `Lifetime` variant or the valid (timing, lifetime) pair + (`crates/types/src/stages/physical/properties.rs`, `PhysicalProperties::new` and + `readable_at`) and its placement in `crates/types/src/ir/properties/timing.rs`, with its data-state edge rules. 2. Offer it in `crates/physical-asap-optimization/src/lib.rs` (the - `MaterializationCandidate` options), after the existing options so default + `MaterializedOutputCandidate` options), after the existing options so default candidate ids stay the same, and gate it on the deployment input that allows it. -3. Price it in `crates/cost/src/usage.rs` (`asap_cost::node_usage`): what it keeps in memory or on +3. Price it in `crates/cost/src/node/usage.rs` (`asap_cost::node_usage`): what it keeps in memory or on disk, and what each evaluation rebuilds. Tests: Stage 2 unit tests for when the option is offered; -`crates/integration-tests/tests/materialization_e2e.rs` and the Example 4 +`crates/integration-tests/tests/physical_properties_e2e.rs` and the Example 4 tests in `planner_layering_example4.rs`. ## 4. Add a Stage 3 check or a deployment input diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 03f3a695c..41c6b5859 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -27,8 +27,8 @@ Inside this workspace, depend on the frontend you need, the stage crates (`asap-logical-asap-optimization` for Stage 1, `asap-physical-asap-optimization` for Stage 2, `asap-plan-selection` for Stage 3), `asap-planner` for the facade that runs the stages, `asap-cost` for the cost model and its inputs (`CostInputs`), -`asap-accuracy` for the accuracy model (`AccuracyModel`, -`AccuracyEvidenceProvider`), and `asap-types`, or on `asap-planner`, which wraps every frontend and +`asap-accuracy` for the accuracy rules and the deployment's measured +accuracy (`AccuracyEvidence`), and `asap-types`, or on `asap-planner`, which wraps every frontend and the planning stages. External users can use Git dependencies pinned to a compatible revision; use the same revision across these crates, and pick a revision at least as new as the APIs you call. Replace `REV` below with a @@ -112,7 +112,7 @@ use asap_types::workload::{ BatchEntry, DataWorkload, DurationMs, Query, PlanningWorkload, QueryLanguage, QueryRequirements, QueryWorkload, }; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; fn main() -> Result<(), Box> { let workload = PlanningWorkload { @@ -187,26 +187,21 @@ asap_plan_selection::select_plan( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], data: &DataWorkload, - accuracy_checks: &AccuracyCheckInputs<'_>, // accuracy model and evidence cost_inputs: &CostInputs, - realizer: &dyn AssignmentRealizer, // Stage 2's, e.g. Stage2AssignmentRealizer -) -> Result +) -> Result<(PhysicalASAPDAG, Selection), ErrorSelectingPlan> ``` | Input or output | Meaning | | --- | --- | -| `PlanningModels::new(accuracy, evidence)` | Your `&dyn AccuracyModel` and `&dyn AccuracyEvidenceProvider`, with default cost inputs and deployment capabilities; `with_accuracy` and `with_evidence` replace one of them | -| `PlanningModels::builtin()` | Built-in accuracy model (`BuiltinAccuracyModel`) and no extra evidence (`NoAccuracyEvidence`). `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | -| `DeploymentCapabilities` (`asap_logical_asap_optimization`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { family: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers `MatAtQueryKeptAcrossEvaluations` for tumbling windows) | -| `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; `provenance` (executor version, machine). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::price_source` says which. The earlier JSON array of `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` still loads) | +| `PlanningModels::new(evidence)` | Your `&dyn AccuracyEvidence` (measured guarantees; `NoMeasuredGuarantees` has none), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | +| `PlanningModels::builtin()` | No measured accuracy evidence. `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | +| `DeploymentCapabilities` (`asap_logical_asap_optimization`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { summary_type: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains) | +| `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; `provenance` (executor version, machine). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::cost_source` says which. The earlier JSON array of `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` still loads) | | `CostModel::UsageOverTime(ResourceUnitCost)` | Cost model 1 (default, `ResourceUnitCost::illustrative()`): unit cost × usage per second; the cheapest valid candidate wins | | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | | `CostModel::ClusterCapacity(ClusterLimits)` | Cost model 3: memory, disk and optional CPU cores are hard limits; `ClusterObjective::{LowestLatency, LeastMemory}` chooses among the candidates that fit | -| `PlanOutput.plans` | One `QueryPlan { entry_index, root }` per query, in `QueryWorkload::entries()` order | -| `PlanOutput.selection` | `Some` with Stage 3's `Selection` under the default pass: `selected_dag`, `nodes_with_proven_non_negative_weights`, `cost` (the selected plan's `PlanCost` with its `ResourceUsage` and `cost_per_second`, `None` under cost model 3), `accepted_accuracy_per_query`, `pruned_candidates` and `search_report` | -| `PlanOutput.tumbling_window_retention_ms` | For each tumbling-window chain the selected plan keeps beyond one evaluation, by node id, how long its tumbling windows are kept (`RealizedPlan::tumbling_window_retention_ms`) | -| `PlanningRun` | `per_computation_candidates`, `logical_asap_candidates`, `physical_asap_candidates`, `selection`, `selected_physical_plan` (`RealizedPlan`), `selected_plan` (the roots to run) and `report` (Pass 2's and Stage 2's counts and each stage's time); `per_computation_candidates` is kept only with `PlanSearchSettings::inspect`; `asap_planner::selected_chosen_candidates` reads the selected plan as one candidate per group of `logical_asap_candidates.memo` | -| `PlanOutput::physical_asap_dag()` | The planned queries as one `PhysicalASAPDAG`, timed with the default `MaterializationAssignment` (see [Export](#export)) | +| `PlanOutput.plan` | The selected plan, one `PhysicalASAPDAG`: `roots` answer the operator queries and `scalar_roots` the standalone scalar queries, in `QueryWorkload::entries()` order; each materialized node carries its `retention_ms`. Its cost is `asap_cost::plan_cost` of it | +| `PlanningRun` | `pass1_memo`, `logical_asap_candidates`, `physical_asap_candidates`, `selection` (the plan and Stage 3's `Selection`: `accepted_accuracy_per_query`, `pruned_candidates`, `milp_variables`, `milp_constraints`, `milp_construction_ms`, `milp_solve_ms`) and `report` (Pass 2's and Stage 2's counts and each stage's time); `pass1_memo` is kept only with `PlanSearchSettings::inspect` | Each candidate's `ResourceUsage` holds its usage without prices: `cpu_ops_per_second`, `scanned_bytes_per_second`, `average_memory_bytes`, @@ -218,7 +213,7 @@ dependencies): ```rust use asap_planner::{e2e_plan, FrontendInput, PlanningModels, UserInput}; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::accuracy::AccuracyTarget; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, DurationMs, PlanningWorkload, PrometheusMetricType, Query, QueryLanguage, QueryRequirements, QueryWorkload, SourceFacts, @@ -310,7 +305,7 @@ query's window comes from its text (range and offset). These facts are separate: `DataWorkload` holds `arrival` (`DataArrival`: `AtRest`, `Ingesting` or `Mixed` with `rows_per_second`, or `Unknown`) and `sources`, a map from PromQL metric or SQL table name to `SourceFacts`: `series_count`, -`metric_type`, `values_non_negative`, `sample_interval` (PromQL and +`metric_type`, `column_signs`, `sample_interval` (PromQL and MetricsQL), `stream_norms`, `max_distinct_per_evaluation` and `distinct_values_per_label` (distinct values per label or column, which plan selection prices a `by (...)` grouping's groups at: the product of its @@ -360,9 +355,9 @@ optional also does not guarantee every planning operation can succeed without it | Function/type | Purpose | | --- | --- | -| `asap_types::ir::apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new())` | Write execution timing into every node from a `MaterializationAssignment` (default: all query time) and validate the data-state edges; `PlanOutput::physical_asap_dag()` applies the default to a planned workload | -| `asap_types::ir::physical_export::compile_physical_asap_dag(&timed_root)` | Export a timed DAG as a `PhysicalASAPDAG`; rejects an untimed node; `compile_physical_asap_workload(&roots)` exports a batch with one root per query; `.validate()` checks it explicitly | -| `PhysicalASAPDAG::materialized_views()` and `.raw_data_retention()` (`asap_types::ir::materialized_view`) | What a runtime keeps for an installed DAG, derived from the DAG and the `retention_ms` installation sets: each materialized view (refresh mode, coverage, inputs, the node's `retention_ms`) and how far back each raw-data source is kept for query-time scans (each query-time `Scan`'s `retention_ms`) | +| `asap_types::ir::apply_physical_properties(&root, &assignment)` | Complete a `PhysicalPropertiesAssignment` (default: all query time) into the physical properties of every node and validate the data-state edges; `apply_physical_properties_to_roots` does it for a workload; `PlanOutput::physical_asap_dag()` applies the pass's `physical_properties` to a planned workload | +| `asap_types::plan::physical_asap_dag::compile_physical_asap_dag(&root, &timed)` | Export a DAG as a `PhysicalASAPDAG`, each node with its properties from the completed assignment `timed`; rejects a node it does not list; `compile_physical_asap_workload(&roots, &timed)` exports a batch with one root per query; `.validate()` checks it explicitly | +| `PhysicalASAPDAG::materialized_views()` and `.raw_data_retention()` (`asap_types::plan::materialized_view`) | What a runtime keeps for an installed DAG, derived from the DAG and the `retention_ms` installation sets: each materialized view (refresh mode, coverage, inputs, the node's `retention_ms`) and how far back each raw-data source is kept for query-time scans (each query-time `Scan`'s `retention_ms`) | Choose the export matching your intended handoff: an untimed semantic DAG is not interchangeable with a versioned execution contract. Preserve diff --git a/docs/develop_docs/physical-compile-coverage.md b/docs/develop_docs/physical-compile-coverage.md index 9d68caec1..a0b221ddf 100644 --- a/docs/develop_docs/physical-compile-coverage.md +++ b/docs/develop_docs/physical-compile-coverage.md @@ -5,7 +5,7 @@ Audience: developers moving computation from ASAPQuery-backend into ## Contract -Logical selection decides what to compute. A `MaterializationAssignment` sets +Logical selection decides what to compute. A `PhysicalPropertiesAssignment` sets node timing (Stage 2 materialization chooses it per physical candidate). `physical_planner::compile` turns a timed `PhysicalASAPDAG` into physical operator DAGs. The backend owns ingestion, tumbling windows, storage, stored-state evaluation, external exact engines, pricing/selection, and execution scheduling. diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index 5fcd74c4c..9a2d23dc5 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -33,8 +33,6 @@ pub struct OperatorNode { pub result_kind: OperatorResultKind, // Relation | InstantVector | RangeVector | State | Scalar pub schema: Schema, // output schema, derived at construction pub guarantee: Option, // None until accuracy assessment establishes one - pub timing: Option, // None until a materialization assignment is applied - pub materialization: Materialization, // NotMaterialized until Stage 2 assigns one } ``` @@ -47,12 +45,11 @@ pub struct OperatorNode { derived (a column reference out of range, a reserved ASAP operator). ASAP planning may retain a more specific schema through `OperatorNode::with_schema`. - `guarantee` is `None` until accuracy assessment establishes one; `None` never means exact. -- `timing` is `None` in every front-end DAG and every candidate. It is written by - `ir::timing::apply_materialization_timings` (see the Post-ASAP IR document); export rejects an - untimed node. -- `materialization` is `NotMaterialized` in every front-end DAG. Stage 2 writes it, with - `timing`, through `ir::timing::apply_materialization_timings` from its materialization - assignment. +- A node has no physical properties (timing, lifetime): a logical candidate has only its + logical properties (#1200). Physical stages keep each node's `PhysicalProperties` in a + `PhysicalPropertiesAssignment` keyed by node; `ir::timing::apply_physical_properties` + completes Stage 2's assignment over every node (see the Post-ASAP IR document), and + export reads it. `OperatorNode::children()` returns the operator's inputs in field order followed by the operator nodes its scalar expressions read (see "Scalar expressions"). Every DAG traversal — diff --git a/docs/develop_docs/target-candidate-api-migration.md b/docs/develop_docs/target-candidate-api-migration.md index 77cc5379d..ec899a98c 100644 --- a/docs/develop_docs/target-candidate-api-migration.md +++ b/docs/develop_docs/target-candidate-api-migration.md @@ -47,10 +47,10 @@ or `Operator::ASAP(ASAPOp)`. Old public names are not kept as aliases. | `SummaryExpr::ValueOperation { .. }` over an evaluation | An ordinary `NonASAPOp` (`Project`, `Filter`, `Sort`, `Limit`, `Aggregate`) reading an ASAP node; `FinalizeExactAccumulator`, `MaintainPopulation`, `EvaluatePopulation` are `ASAPOp` variants | | `Replacement::Summary(..)` / `Replacement::Rewrite(..)` | `Replacement::SubDAG(Rc)`; `is_logical_rewrite` tells them apart | | `SummaryFamilyType` | `FieldDataType` (its non-`Plain` variants) | -| Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_materialization_timings` writes it from a `MaterializationAssignment` (default: all query time) | +| Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_physical_properties` writes it from a `PhysicalPropertiesAssignment` (default: all query time) | | `UnresolvedQueryExpr` + `asap_types::pre_asap::resolve_root` | `UnresolvedOp` / `UnresolvedScalar` + `asap_frontend_common::resolve_root` | | `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `asap_frontend_common` `canonicalize` (crate-private), `share_common_sub_dags` (crate-private in `asap_logical_asap_optimization`, `stage0/cse.rs`) | -| `asap_types::post_asap::compile_post_asap_dag` (wire version 5, `Fallback`/`Binary`/`Value` payloads) | `asap_types::ir::physical_export::compile_physical_asap_dag` (`PhysicalASAPDAG`, carried in the plan file versioned by `asap_planner::INSTALLABLE_PLAN_WIRE_VERSION`: one node per operator, `Operator` payloads, `ScalarRef` edges); input must be timed | +| `asap_types::post_asap::compile_post_asap_dag` (wire version 5, `Fallback`/`Binary`/`Value` payloads) | `asap_types::plan::physical_asap_dag::compile_physical_asap_dag` (`PhysicalASAPDAG`, carried in the plan file versioned by `asap_planner::INSTALLABLE_PLAN_WIRE_VERSION`: one node per operator, `Operator` payloads, `ScalarRef` edges); input must be timed | | Exported schema JSON `columns` | `fields` | Field and schema details: [Pre-ASAP IR](pre-asap-ir.md) and diff --git a/docs/develop_docs/tumbling-window-search-compression-evaluation.md b/docs/develop_docs/tumbling-window-search-compression-evaluation.md index 2419cf97e..5ddcdfc26 100644 --- a/docs/develop_docs/tumbling-window-search-compression-evaluation.md +++ b/docs/develop_docs/tumbling-window-search-compression-evaluation.md @@ -110,7 +110,7 @@ The nine viewer workloads also passed **9/9**, with **40/40** query requirements ### Final regression follow-up -The regression suite found a genuine optimization regression outside window planning. Two identical SQL percentiles over an at-rest table were sent through progressive tumbling window rounds solely because they repeat every ten minutes. No computation had a tumbling-window-eligible window, so the rounds rebuilt the same 40-configuration, 150-connection model and restarted refinement. The 128-evaluation budget expired with two identical unmaterialized KLL builds, costing 0.007678692288679677 per second instead of the baseline's one shared query-materialized build at 0.006000017830006526 (+27.98%). The original pointer-sharing assertion failed both in the suite and in isolation; it was retained unchanged. +The regression suite found a genuine optimization regression outside window planning. Two identical SQL percentiles over an at-rest table were sent through progressive tumbling window rounds solely because they repeat every ten minutes. No computation had a tumbling-window-eligible window, so the rounds rebuilt the same 40-configuration, 150-connection model and restarted refinement. The 128-evaluation budget expired with two identical not_materialized KLL builds, costing 0.007678692288679677 per second instead of the baseline's one shared query-materialized build at 0.006000017830006526 (+27.98%). The original pointer-sharing assertion failed both in the suite and in isolation; it was retained unchanged. Revision `fe171f64` bypasses tumbling-window-domain expansion when the initial computations contain neither a positive range window nor an existing tumbling-window chain. It searches the complete candidate domain directly; at-rest status alone does not disable tumbling window planning. The same SQL fixture now passes the original sharing test and selects one shared KLL with exactly the baseline cost. Its report records 58 evaluations, `optimal`, one `Complete` round, and no pending widths. This fix avoids repeated identical-domain searches rather than increasing the budget or relaxing correctness checks. diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index c6800ee1c..f713f4a0f 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -9,7 +9,7 @@ document shows: - **Stage 1**, the logical candidates of the memo and the selected plan's logical DAG (`L1`); - **Stage 2**, the selected plan as an executable physical DAG (`P1`), with - each node's timing and materialization, and the physical candidates Stage 2 + each node's timing and lifetime, and the physical candidates Stage 2 pruned; - **Stage 3**, the selected plan's cost, the accuracy it accepted per query, its materialized views and the search report. @@ -78,7 +78,7 @@ To run it, see [RUNNING.md](RUNNING.md). candidates are filled green with `✓ selected`. Clicking a candidate or a group shows its nodes, children, parents and queries in the details. - **Details**: click a node for its operator, the queries reaching it, - timing, materialization, output schema, guarantee and Stage 3 cost; click + timing, lifetime, output schema, guarantee and Stage 3 cost; click an edge for its schema and data state. Other documents open with **Open stage document or workload…**, by dropping diff --git a/tools/dag-viewer/app.js b/tools/dag-viewer/app.js index 3c1b4e877..34aed44c5 100644 --- a/tools/dag-viewer/app.js +++ b/tools/dag-viewer/app.js @@ -124,7 +124,7 @@ if (d.tumblingWindowOf !== undefined) items.push(['tumbling window', `${d.tumblingWindowIndex + 1} of ${body.tumbling_window_count} of SummaryTumblingWindowChain ${d.tumblingWindowOf}, window ${tumblingWindowTimeText(d.tumbling_window, body.tumbling_window_width_ms)}`]); if (d.expandable) items.push(['tumbling windows', `${body.tumbling_window_count} × ${durationText(body.tumbling_window_width_ms)} (click the chain to show them, a tumbling window to hide them)`]); if (d.timing !== undefined) items.push(['runs at', formatTiming(d.timing)]); - if (n.materialization) items.push(['materialization', compactWire(n.materialization)]); + if (n.lifetime) items.push(['lifetime', compactWire(n.lifetime)]); if (n.output_state && n.output_state.primitive) items.push(['output primitive', compactWire(n.output_state.primitive)]); if (d.cost) { const value = typeof d.cost.cost_per_second === 'number' @@ -352,9 +352,9 @@ $('views-meta').innerHTML = selection.materialized_views_error ? `no installable plan: ${esc(selection.materialized_views_error)}` : `${count} materialized view${count === 1 ? '' : 's'} in ${groups.length} group${groups.length === 1 ? '' : 's'} of ${esc(selection.selected_candidate)}` + - (count ? '' : ': nothing is kept across evaluations; every query computes at query time'); + (count ? '' : ': nothing is materialized across evaluations; every query computes at query time'); const tumblingWindowsCell = (g) => (g.tumblingWindows.length > 1 - ? `
${esc(g.tumblingWindows.length)} installed nodes
    ${g.tumblingWindows.map((p) => `
  • node ${esc(p.nodeId)} · ${esc(p.tumblingWindow)} · window ${esc(p.window)} · kept ${esc(p.retention)}
  • `).join('')}
` + ? `
${esc(g.tumblingWindows.length)} installed nodes
    ${g.tumblingWindows.map((p) => `
  • node ${esc(p.nodeId)} · ${esc(p.tumblingWindow)} · window ${esc(p.window)} · materialized ${esc(p.retention)}
  • `).join('')}
` : `node ${esc(g.tumblingWindows[0].nodeId)} · window ${esc(g.tumblingWindows[0].window)}`); $('materializedViews').innerHTML = groups.length ? table( ['Stage 2 node', 'definition', 'kind', 'tumbling windows', 'refresh', 'retention', 'installed', 'Stage 3 cost', 'read by'], diff --git a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-1h-60s-300s.json b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-1h-60s-300s.json index f6d18e95d..c53e1cc4d 100644 --- a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-1h-60s-300s.json +++ b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-1h-60s-300s.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-q09-1h.json b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-q09-1h.json index cbbb225a3..cbcd46e14 100644 --- a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-q09-1h.json +++ b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-q09-1h.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q09-15m-1h.json b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q09-15m-1h.json index 0836b9c93..0666ce616 100644 --- a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q09-15m-1h.json +++ b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q09-15m-1h.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/promql/rangeheavy-t7-t4-1h.json b/tools/dag-viewer/memo-examples/promql/rangeheavy-t7-t4-1h.json index 35ab437f7..f1b9fe3a0 100644 --- a/tools/dag-viewer/memo-examples/promql/rangeheavy-t7-t4-1h.json +++ b/tools/dag-viewer/memo-examples/promql/rangeheavy-t7-t4-1h.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/promql/rangeheavy-t8-t5-1h.json b/tools/dag-viewer/memo-examples/promql/rangeheavy-t8-t5-1h.json index c3b6d73b5..47eab5eb9 100644 --- a/tools/dag-viewer/memo-examples/promql/rangeheavy-t8-t5-1h.json +++ b/tools/dag-viewer/memo-examples/promql/rangeheavy-t8-t5-1h.json @@ -72,7 +72,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json b/tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json index af180ad3b..ddf8d3755 100644 --- a/tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json +++ b/tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json @@ -33,7 +33,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/sql/rollupheavy-t11-service-endpoint-5m.json b/tools/dag-viewer/memo-examples/sql/rollupheavy-t11-service-endpoint-5m.json index 9da4f70c9..61566c302 100644 --- a/tools/dag-viewer/memo-examples/sql/rollupheavy-t11-service-endpoint-5m.json +++ b/tools/dag-viewer/memo-examples/sql/rollupheavy-t11-service-endpoint-5m.json @@ -55,7 +55,7 @@ "http": { "series_count": 10000, "sample_interval": 5, - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "max_distinct_per_evaluation": 1000000 } } diff --git a/tools/dag-viewer/memo.js b/tools/dag-viewer/memo.js index 5216dbab3..d13aa801b 100644 --- a/tools/dag-viewer/memo.js +++ b/tools/dag-viewer/memo.js @@ -50,11 +50,11 @@ const memoIsNode = (nodes, id) => Number.isInteger(id) && id >= 0 && id < nodes. // One line per entry of a stage's `pruned_candidates` on group `group`: // Pass 1's `candidate`, or Stage 2's operator with its timing, -// materialization and scope copy; then the reason. 'none' without any. +// lifetime and scope copy; then the reason. 'none' without any. function memoPrunedText(entries, group) { const lines = (Array.isArray(entries) ? entries : []).filter((p) => p && p.group === group).map((p) => { const what = 'candidate' in p ? p.candidate - : `node ${p.operator_node} ${p.operator} ${p.timing} ${JSON.stringify(p.materialization)}`; + : `node ${p.operator_node} ${p.operator} ${p.timing} ${JSON.stringify(p.lifetime)}`; return `${what}: ${p.reason}`; }); return lines.length ? lines.join('; ') : 'none'; diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index aff3da8fa..131a02e27 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -224,7 +224,7 @@ function stageDag(dag, physical) { const [kind, body] = stageOperatorParts(node.payload); return { id: node.id, kind, body, schema: node.output_schema, guarantee: node.guarantee, - materialization: node.materialization, timing: node.output_state ? node.output_state.timing : undefined, raw: node, + lifetime: node.lifetime, timing: node.output_state ? node.output_state.timing : undefined, raw: node, }; }); const edges = dag.edges.map((edge) => ({ producer: edge.producer, consumer: edge.consumer, raw: edge })); @@ -241,8 +241,7 @@ function stageDag(dag, physical) { const [kind, body] = stageOperatorParts(node.operator); stageLogicalChildren(body).forEach((producer) => edges.push({ producer, consumer: id, raw: null })); return { - id, kind, body, schema: node.schema, guarantee: node.guarantee, - materialization: node.materialization, timing: node.timing === null ? undefined : node.timing, raw: node, + id, kind, body, schema: node.schema, guarantee: node.guarantee, raw: node, }; }); const queries = dag.roots.map((root) => ('Scalar' in root ? { scalar: root.Scalar } : { node: root.Operator })); @@ -321,7 +320,6 @@ function uniqueIds(list, where, errors) { // it is not priced. const FAILED_CHECK_CAUSES = [ 'accuracy_target_missed', - 'count_min_weights_not_proven_non_negative', 'no_accuracy_guarantee', 'latency_bound_exceeded', 'latency_over_repeat_interval', 'over_cluster_capacity', 'infeasible', 'cost_not_representable', ]; @@ -1036,7 +1034,7 @@ function dataWorkloadView(dataWorkload, queries) { name, series: typeof facts.series_count === 'number' ? facts.series_count : '—', type: facts.metric_type || '—', - nonNegative: facts.values_non_negative ? 'yes' : String(facts.metric_type).toLowerCase() === 'counter' ? 'yes (counter)' : '—', + nonNegative: (facts.column_signs || {}).value === 'non_negative' ? 'yes' : String(facts.metric_type).toLowerCase() === 'counter' ? 'yes (counter)' : '—', sampleInterval: facts.sample_interval === undefined || facts.sample_interval === null ? '—' : durationText(facts.sample_interval), streamNorms: norms(facts.stream_norms), maxDistinct: typeof facts.max_distinct_per_evaluation === 'number' ? facts.max_distinct_per_evaluation : '—', @@ -1079,7 +1077,7 @@ function materializedViewRows(doc) { definition: output.coverage.definition_id, kind: summary ? `${kind} · ${summary}` : kind, tumblingWindow, - refresh: output.refresh === 'at_ingestion' ? 'at ingestion' : 'on first read (kept across evaluations)', + refresh: output.refresh === 'at_ingestion' ? 'at ingestion' : 'on first read (materialized across evaluations)', inputs: `${output.inputs.raw_data_windows.length} raw-data window(s)` + (output.inputs.materialized_views.length ? `, views ${output.inputs.materialized_views.join(', ')}` : ''), retention: durationText(output.retention_ms), @@ -1171,7 +1169,7 @@ function usageText(usage) { function deploymentText(capabilities) { if (!capabilities || typeof capabilities !== 'object') return ''; const summaries = Array.isArray(capabilities.supported_summaries) - ? `summaries: ${capabilities.supported_summaries.map((s) => `${s.family} (${(s.readouts || []).join(', ')})`).join(', ')}` + ? `summaries: ${capabilities.supported_summaries.map((s) => `${s.summary_type} (${(s.readouts || []).join(', ')})`).join(', ')}` : 'every summary the planner builds'; return [capabilities.raw_data_retained === false ? 'raw data not retained' : 'raw data retained', capabilities.query_time_results_retained ? 'query-time results retained' : 'query-time results not retained', summaries].join(' · '); diff --git a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json index 633a16fa6..a5bcc3476 100644 --- a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json +++ b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} diff --git a/tools/dag-viewer/test-fixtures/memo-small.json b/tools/dag-viewer/test-fixtures/memo-small.json index cf7f96aea..2b8af0eb1 100644 --- a/tools/dag-viewer/test-fixtures/memo-small.json +++ b/tools/dag-viewer/test-fixtures/memo-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}} diff --git a/tools/dag-viewer/workload-sets.js b/tools/dag-viewer/workload-sets.js index ece483103..28bf710da 100644 --- a/tools/dag-viewer/workload-sets.js +++ b/tools/dag-viewer/workload-sets.js @@ -19,7 +19,7 @@ const parts = []; if (capabilities.raw_data_retained === false) parts.push('raw data not retained'); if (capabilities.query_time_results_retained === true) parts.push('query-time results retained'); - if (Array.isArray(capabilities.supported_summaries)) parts.push(`${capabilities.supported_summaries.length} supported summary families`); + if (Array.isArray(capabilities.supported_summaries)) parts.push(`${capabilities.supported_summaries.length} supported summary types`); return parts.join(', ') || 'default deployment capabilities'; } diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-01.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-01.json index 05a8bddd0..8480e3aa9 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-01.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-01.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-02.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-02.json index a32fb624e..23fde7efe 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-02.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-02.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-03.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-03.json index 42cb5cebb..90ad67f7c 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-03.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-03.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-04.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-04.json index e7a49f676..42fcd0d1b 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-04.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-04.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-05.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-05.json index 97d5c8a7f..4693f5b31 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-05.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-05.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-06.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-06.json index bad5fb16d..15574d528 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-06.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-06.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-07.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-07.json index 690d9b987..b79ccfab7 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-07.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-07.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-08.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-08.json index 951365d04..ded627a08 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-08.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-08.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-09.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-09.json index 9c55b9869..1e5bbd1df 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-09.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-09.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-10.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-10.json index 50ec85402..4937890e3 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-10.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-10.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-01.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-01.json index 3c6a1258d..f61726db0 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-01.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-01.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-02.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-02.json index 29bc69aaa..85913b6d4 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-02.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-02.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-03.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-03.json index 563bf1baf..162dafc10 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-03.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-03.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-04.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-04.json index 176dca915..2ab1953f1 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-04.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-04.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-05.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-05.json index 0100b98c7..5a604e05c 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-05.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-05.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-06.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-06.json index 36f4f9cef..fc267e744 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-06.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-06.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-07.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-07.json index 55e7850df..5de0750f0 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-07.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-07.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-08.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-08.json index 0883c7b99..3ed4aa86d 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-08.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-08.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-09.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-09.json index ba5415bde..b04a04c86 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-09.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-09.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-10.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-10.json index 9bcfede93..0830cf521 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-10.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-10.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-01.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-01.json index 966256bd4..838970ad6 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-01.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-01.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-02.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-02.json index 099306fe2..928638d74 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-02.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-02.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-03.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-03.json index 662ddbdf5..2b397b08b 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-03.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-03.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-04.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-04.json index f6a18db5d..25c3318f9 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-04.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-04.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-05.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-05.json index e1e63aa36..da1813dc4 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-05.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-05.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-06.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-06.json index cd0c15a80..99e20ad82 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-06.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-06.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-07.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-07.json index 771acf36c..f88ce4219 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-07.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-07.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-08.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-08.json index 9db1b41d3..dfcebffd6 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-08.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-08.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-09.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-09.json index c95ec5037..8fdcc8a7d 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-09.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-09.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-10.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-10.json index adc66bfdd..2773378c6 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-10.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-10.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null